06:33:27.701 [INFO - main] (KuduTestHarness.java:155) Creating a new MiniKuduCluster...
06:33:27.701 [INFO - main] (TempDirUtils.java:71) Using the temp directory defined by TEST_TMPDIR: /tmp/dist-test-taskbZXLUY/test-tmp
06:33:27.701 [INFO - main] (KuduBinaryLocator.java:83) Using Kudu binary directory specified by system property 'kuduBinDir': ../../../build/asan/bin
06:33:27.701 [INFO - main] (MiniKuduCluster.java:217) Starting process: [/tmp/dist-test-taskbZXLUY/java/build/dist-test/../../../build/asan/bin/kudu, test, mini_cluster, --serialization=pb]
06:33:27.702 [DEBUG - main] (MiniKuduCluster.java:185) Request: create_cluster {
num_masters: 3
num_tservers: 3
enable_kerberos: false
cluster_root: "/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193"
extra_master_flags: "--txn_manager_enabled"
extra_master_flags: "--raft_heartbeat_interval_ms=100"
extra_tserver_flags: "--txn_keepalive_interval_ms=1000"
extra_tserver_flags: "--txn_staleness_tracker_interval_ms=250"
extra_tserver_flags: "--enable_txn_system_client_init=true"
hms_mode: NONE
mini_kdc_options {
}
principal: "kudu"
mini_oidc_options {
}
}
06:33:28.015 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:33:28.016 [DEBUG - main] (MiniKuduCluster.java:185) Request: start_cluster {
}
06:33:28.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.017899 24199 mini_oidc.cc:152] Starting JWKS server
06:33:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.020330 24199 webserver.cc:460] Webserver started at http://127.0.0.1:41539/ using document root <none> and password file <none>
06:33:28.020 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.020520 24199 mini_oidc.cc:189] Starting OIDC Discovery server
06:33:28.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.021051 24199 webserver.cc:460] Webserver started at http://127.0.0.1:33649/ using document root <none> and password file <none>
06:33:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T06:33:28Z chronyd version 3.5 starting (+CMDMON +NTP +REFCLOCK +RTC -PRIVDROP -SCFILTER -SIGND +ASYNCDNS -SECHASH -IPV6 +DEBUG)
06:33:28.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T06:33:28Z Disabled control of system clock
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.053453 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/logs
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data/info.pb
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:28.053 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.254:38015
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.254
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
06:33:28.054 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
06:33:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.350028 24212 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.350317 24212 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:28.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.350399 24212 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357311 24212 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357414 24212 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357451 24212 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357499 24212 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357539 24212 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357590 24212 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:33:28.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.357640 24212 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41539/jwks.json
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.366652 24212 master_runner.cc:386] Master server non-default flags:
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.254:38015
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data/info.pb
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.254
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/logs
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:28.367 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:28.368 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.367902 24212 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:28.369 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.369619 24212 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.381101 24217 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.381062 24221 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.381147 24218 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.385 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.385456 24212 server_base.cc:1034] running on GCE node
06:33:28.386 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.386138 24212 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:28.387 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.387493 24212 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:28.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.388725 24212 hybrid_clock.cc:648] HybridClock initialized: now 1786257208388692 us; error 81 us; skew 500 ppm
06:33:28.389 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.389103 24212 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:28.391 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.391342 24212 webserver.cc:460] Webserver started at http://127.23.161.254:36637/ using document root <none> and password file <none>
06:33:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.392050 24212 fs_manager.cc:359] Metadata directory not provided
06:33:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.392228 24212 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:28.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.392567 24212 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.395140 24212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data/instance:
06:33:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ceef4bfa72a4468aad391f4034246263"
06:33:28.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.395968 24212 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal/instance:
06:33:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ceef4bfa72a4468aad391f4034246263"
06:33:28.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.400997 24212 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.000s
06:33:28.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.404175 24228 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.405333 24212 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.002s sys 0.001s
06:33:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.405535 24212 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal
06:33:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "ceef4bfa72a4468aad391f4034246263"
06:33:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.405735 24212 fs_report.cc:383] FS layout report
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data/data
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:28.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:28.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.431912 24212 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:28.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.432814 24212 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:28.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.433152 24212 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.454483 24212 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.254:38015
06:33:28.454 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.454499 24293 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.254:38015 every 8 connection(s)
06:33:28.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.455955 24212 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/data/info.pb
06:33:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.459244 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24212
06:33:28.459 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.459530 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-0/wal/instance
06:33:28.460 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.460248 24294 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.462428 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/logs
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:28.462 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.253:45177
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.253
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
06:33:28.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
06:33:28.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.469434 24294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } has no permanent_uuid. Determining permanent_uuid...
06:33:28.487 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.486989 24294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } has no permanent_uuid. Determining permanent_uuid...
06:33:28.488 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.488035 24232 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.489 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.489524 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.538883 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } attempt: 1
06:33:28.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.540807 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.609135 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } attempt: 2
06:33:28.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.611202 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.702 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.702487 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } attempt: 3
06:33:28.704 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.704341 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.775152 24297 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.775408 24297 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:28.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.775483 24297 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.781970 24297 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782051 24297 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782088 24297 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782135 24297 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782183 24297 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782234 24297 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:33:28.782 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.782265 24297 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41539/jwks.json
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.790819 24297 master_runner.cc:386] Master server non-default flags:
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.253:45177
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.253
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/logs
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:28.791 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:28.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.792022 24297 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:28.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.793725 24297 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.805310 24308 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.805353 24307 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.805 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.805486 24310 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:28.806 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.806285 24297 server_base.cc:1034] running on GCE node
06:33:28.807 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.806944 24297 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:28.808 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.808161 24297 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.809381 24297 hybrid_clock.cc:648] HybridClock initialized: now 1786257208809323 us; error 44 us; skew 500 ppm
06:33:28.809 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.809787 24297 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:28.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.812141 24297 webserver.cc:460] Webserver started at http://127.23.161.253:46215/ using document root <none> and password file <none>
06:33:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.812945 24297 fs_manager.cc:359] Metadata directory not provided
06:33:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.813078 24297 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:28.813 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.813414 24297 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.815973 24297 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/instance:
06:33:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.816831 24297 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal/instance:
06:33:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:28.817 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.821 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.821547 24297 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.004s sys 0.000s
06:33:28.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.824904 24316 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.826047 24297 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.826223 24297 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.826431 24297 fs_report.cc:383] FS layout report
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/data
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:28.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:28.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:28.843 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.843219 24297 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:28.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.844038 24297 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:28.844 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.844405 24297 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:28.847 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.847724 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } attempt: 4
06:33:28.849 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.849757 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:28.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.865147 24297 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.253:45177
06:33:28.865 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.865177 24388 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.253:45177 every 8 connection(s)
06:33:28.866 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.866394 24297 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:28.870 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.869956 24389 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:28.873 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.873843 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24297
06:33:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.873991 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal/instance
06:33:28.874 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.874809 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/logs
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.252:42373
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.252
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
06:33:28.875 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
06:33:28.876 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.876058 24389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } has no permanent_uuid. Determining permanent_uuid...
06:33:28.890 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.890735 24389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } has no permanent_uuid. Determining permanent_uuid...
06:33:28.900 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.900787 24389 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } has no permanent_uuid. Determining permanent_uuid...
06:33:28.901 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.901675 24321 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:28.903 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.903365 24389 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:28.953 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:28.952721 24389 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 1
06:33:28.954 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:28.954790 24389 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.022994 24389 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 2
06:33:29.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.025030 24389 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.116 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.116283 24389 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 3
06:33:29.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.118448 24389 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.149 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.149044 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } attempt: 5
06:33:29.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.156579 24294 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } has no permanent_uuid. Determining permanent_uuid...
06:33:29.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.158236 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.200021 24393 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:29.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.200417 24393 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:29.200 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.200533 24393 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207239 24393 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207362 24393 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207415 24393 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207525 24393 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207577 24393 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207648 24393 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:33:29.207 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.207748 24393 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41539/jwks.json
06:33:29.209 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.209450 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 1
06:33:29.211 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.211344 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.217351 24393 master_runner.cc:386] Master server non-default flags:
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.252:42373
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:29.217 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.252
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/logs
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:29.218 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.218858 24393 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:29.221 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.220911 24393 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:29.232 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.232599 24400 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.232539 24403 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.232918 24401 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:29.233 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.233623 24393 server_base.cc:1034] running on GCE node
06:33:29.234 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.234391 24393 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:29.235 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.235647 24393 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:29.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.236881 24393 hybrid_clock.cc:648] HybridClock initialized: now 1786257209236838 us; error 28 us; skew 500 ppm
06:33:29.237 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.237246 24393 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:29.239 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.239331 24393 webserver.cc:460] Webserver started at http://127.23.161.252:37135/ using document root <none> and password file <none>
06:33:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.239962 24393 fs_manager.cc:359] Metadata directory not provided
06:33:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.240084 24393 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:29.240 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.240463 24393 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.242882 24393 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/instance:
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d573f1a12d4c4086bbf20f8e73481ad7"
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:29 on dist-test-slave-1jjb"
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.243660 24393 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal/instance:
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d573f1a12d4c4086bbf20f8e73481ad7"
06:33:29.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:29 on dist-test-slave-1jjb"
06:33:29.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.248337 24393 fs_manager.cc:691] Time spent creating directory manager: real 0.004s user 0.003s sys 0.004s
06:33:29.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.251483 24410 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:29.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.252725 24393 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.003s sys 0.000s
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.252924 24393 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d573f1a12d4c4086bbf20f8e73481ad7"
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:29 on dist-test-slave-1jjb"
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.253147 24393 fs_report.cc:383] FS layout report
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/data
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:29.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:29.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:29.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.261675 24389 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 4
06:33:29.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.263437 24389 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.265817 24393 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.266463 24393 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:29.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.266688 24393 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:29.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.279547 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 2
06:33:29.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.281618 24294 consensus_peers.cc:646] Error getting permanent uuid from config peer 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:29.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.288319 24393 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.252:42373
06:33:29.288 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.288331 24474 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.252:42373 every 8 connection(s)
06:33:29.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.289538 24393 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:29.293 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.293087 24475 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 00000000000000000000000000000000. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:29.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.296303 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24393
06:33:29.296 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.296437 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal/instance
06:33:29.300 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.300042 24475 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } has no permanent_uuid. Determining permanent_uuid...
06:33:29.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.326970 24475 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } has no permanent_uuid. Determining permanent_uuid...
06:33:29.334 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.334200 24475 sys_catalog.cc:422] member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } has no permanent_uuid. Determining permanent_uuid...
06:33:29.343 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.343344 24475 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Bootstrap starting.
06:33:29.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.346242 24475 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Neither blocks nor log segments found. Creating new log.
06:33:29.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.347173 24475 log.cc:824] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Log is configured to *not* fsync() on all Append() calls
06:33:29.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.350318 24475 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: No bootstrap required, opened a new log
06:33:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.358312 24475 raft_consensus.cc:348] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.358624 24475 raft_consensus.cc:374] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:29.358 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.358706 24475 raft_consensus.cc:729] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: d573f1a12d4c4086bbf20f8e73481ad7, State: Initialized, Role: FOLLOWER
06:33:29.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.359220 24475 consensus_queue.cc:260] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.360584 24483 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.360965 24483 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.361 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.361691 24475 sys_catalog.cc:564] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: configured and running, proceeding with master startup.
06:33:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.371412 24498 catalog_manager.cc:1558] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:33:29.371 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.371517 24498 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:33:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.367795 24393 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: OK, instance_id { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" instance_seqno: 1786257208449109 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:33:29.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.375754 24393 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:29.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.387858 24294 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 3
06:33:29.397 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.397780 24294 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: Bootstrap starting.
06:33:29.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.401180 24294 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: Neither blocks nor log segments found. Creating new log.
06:33:29.402 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.402190 24294 log.cc:824] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: Log is configured to *not* fsync() on all Append() calls
06:33:29.404 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.404862 24294 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: No bootstrap required, opened a new log
06:33:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.411715 24294 raft_consensus.cc:348] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.412003 24294 raft_consensus.cc:374] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:29.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.412163 24294 raft_consensus.cc:729] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: ceef4bfa72a4468aad391f4034246263, State: Initialized, Role: FOLLOWER
06:33:29.413 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.412818 24294 consensus_queue.cc:260] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.413972 24501 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.414435 24501 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.415261 24294 sys_catalog.cc:564] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: configured and running, proceeding with master startup.
06:33:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.424392 24512 catalog_manager.cc:1558] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: loading cluster ID for follower catalog manager: Not found: cluster ID entry not found
06:33:29.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.424501 24512 catalog_manager.cc:873] Not found: cluster ID entry not found: failed to prepare follower catalog manager, will retry
06:33:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.423796 24212 master_runner.cc:186] Error getting master registration for 127.23.161.253:45177: OK, instance_id { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" instance_seqno: 1786257208860187 } error { code: CATALOG_MANAGER_NOT_INITIALIZED status { code: SERVICE_UNAVAILABLE message: "Catalog manager is not initialized. State: Starting" } }
06:33:29.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.433205 24212 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:29.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.435593 24234 tablet.cc:2369] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:33:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.438503 24483 raft_consensus.cc:482] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:29.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.438789 24483 raft_consensus.cc:504] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.440 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.440160 24483 leader_election.cc:288] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177)
06:33:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.440865 24256 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ceef4bfa72a4468aad391f4034246263" is_pre_election: true
06:33:29.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.441418 24256 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d573f1a12d4c4086bbf20f8e73481ad7 in term 0.
06:33:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.442127 24412 leader_election.cc:302] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7; no voters:
06:33:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.442534 24483 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:33:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.442669 24483 raft_consensus.cc:482] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:33:29.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.442776 24483 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 0 FOLLOWER]: Advancing to term 1
06:33:29.446 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.445834 24483 raft_consensus.cc:504] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.447947 24256 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ceef4bfa72a4468aad391f4034246263"
06:33:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.448045 24483 leader_election.cc:288] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 1 election: Requested vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177)
06:33:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.448173 24256 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 0 FOLLOWER]: Advancing to term 1
06:33:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.450233 24256 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d573f1a12d4c4086bbf20f8e73481ad7 in term 1.
06:33:29.450 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.450642 24412 leader_election.cc:302] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7; no voters:
06:33:29.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.450947 24483 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Leader election won for term 1
06:33:29.463 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.463758 24483 raft_consensus.cc:686] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 LEADER]: Becoming Leader. State: Replica: d573f1a12d4c4086bbf20f8e73481ad7, State: Running, Role: LEADER
06:33:29.464 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.464566 24483 consensus_queue.cc:237] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.469519 24515 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d573f1a12d4c4086bbf20f8e73481ad7. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.470 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.469965 24515 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: LEADER
06:33:29.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.476116 24517 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:33:29.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.481330 24517 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:33:29.498 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.498291 24256 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Refusing update from remote peer d573f1a12d4c4086bbf20f8e73481ad7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:33:29.499 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.499143 24483 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.518636 24501 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d573f1a12d4c4086bbf20f8e73481ad7. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.518968 24501 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.521 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:29.519322 24416 tablet.cc:2369] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:33:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.524957 24501 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.525177 24501 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.536 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.536598 24483 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.536898 24483 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: LEADER
06:33:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.538300 24517 catalog_manager.cc:1347] Generated new cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:29.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.538406 24517 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:33:29.563 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.562644 24389 consensus_peers.cc:656] Retrying to get permanent uuid for remote peer: member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } attempt: 5
06:33:29.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.576243 24517 catalog_manager.cc:1370] Generated new certificate authority record
06:33:29.580 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.579284 24517 catalog_manager.cc:1504] Loading token signing keys...
06:33:29.581 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.581089 24389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Bootstrap starting.
06:33:29.588 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.587919 24389 tablet_bootstrap.cc:654] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Neither blocks nor log segments found. Creating new log.
06:33:29.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.589485 24389 log.cc:824] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Log is configured to *not* fsync() on all Append() calls
06:33:29.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.601743 24517 catalog_manager.cc:5868] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Generated new TSK 0
06:33:29.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.603475 24389 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: No bootstrap required, opened a new log
06:33:29.604 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.603924 24517 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:33:29.611 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.611795 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/logs
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data/info.pb
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.193:0
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.193
06:33:29.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.193
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:29.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
06:33:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.627637 24389 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.628065 24389 raft_consensus.cc:374] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:29.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.628202 24389 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb210d252711436ab28b2ae64acb48a3, State: Initialized, Role: FOLLOWER
06:33:29.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.628894 24389 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.645018 24527 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:29.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.646924 24347 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:29.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.647089 24527 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:29.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.647652 24348 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "eb210d252711436ab28b2ae64acb48a3" is_pre_election: true
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.648860 24347 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 0 FOLLOWER]: Advancing to term 1
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.646874 24389 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: configured and running, proceeding with master startup.
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.650923 24255 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb210d252711436ab28b2ae64acb48a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "ceef4bfa72a4468aad391f4034246263" is_pre_election: true
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.651770 24527 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.651969 24347 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d573f1a12d4c4086bbf20f8e73481ad7 in term 1.
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.652240 24527 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.652499 24527 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.654158 24346 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Refusing update from remote peer d573f1a12d4c4086bbf20f8e73481ad7: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:33:29.656 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.655123 24515 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:29.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.671834 24437 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb210d252711436ab28b2ae64acb48a3" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" is_pre_election: true
06:33:29.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.673055 24321 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:29.692 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.691669 24526 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:33:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.692898 24527 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d573f1a12d4c4086bbf20f8e73481ad7. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.693198 24527 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.693 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.693640 24534 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.694073 24534 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:29.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.717833 24544 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:29.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.718189 24544 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: loading cluster ID for follower catalog manager: success
06:33:29.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.721840 24544 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: acquiring CA information for follower catalog manager: success
06:33:29.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.730822 24544 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:33:29.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.739645 24523 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:29.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:29.740052 24523 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: LEADER
06:33:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.098080 24525 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.098716 24525 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:30.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.098853 24525 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.107039 24525 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.107230 24525 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
06:33:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.107291 24525 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
06:33:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.107375 24525 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
06:33:30.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.107496 24525 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.161.193
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.118969 24525 tablet_server_runner.cc:78] Tablet server non-default flags:
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.193:0
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data/info.pb
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.193
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.193
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/logs
06:33:30.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:30.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.120577 24525 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:30.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.122509 24525 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.137038 24550 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.137033 24551 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.138248 24553 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.138563 24525 server_base.cc:1034] running on GCE node
06:33:30.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.139118 24525 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:30.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.140044 24525 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.141299 24525 hybrid_clock.cc:648] HybridClock initialized: now 1786257210141252 us; error 42 us; skew 500 ppm
06:33:30.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.141551 24525 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.143913 24525 webserver.cc:460] Webserver started at http://127.23.161.193:35807/ using document root <none> and password file <none>
06:33:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.144553 24525 fs_manager.cc:359] Metadata directory not provided
06:33:30.144 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.144631 24525 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:30.145 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.144874 24525 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.147416 24525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data/instance:
06:33:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:30.147 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.148254 24525 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal/instance:
06:33:30.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:30.148 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.153 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.153664 24525 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.004s sys 0.000s
06:33:30.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.157092 24559 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.158448 24525 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.158576 24525 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.158733 24525 fs_report.cc:383] FS layout report
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal
06:33:30.158 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data/data
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:30.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:30.169 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.169368 24525 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:30.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.170028 24525 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:30.170 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.170276 24525 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.173159 24525 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:33:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.173267 24525 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:33:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.173388 24525 ts_tablet_manager.cc:610] Registered 0 tablets
06:33:30.173 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.173448 24525 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:33:30.263 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.263756 24525 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.193:37599
06:33:30.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.263836 24701 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.193:37599 every 8 connection(s)
06:33:30.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.265076 24525 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/data/info.pb
06:33:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.267549 24703 heartbeater.cc:347] Connected to a master server at 127.23.161.254:38015
06:33:30.268 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.267808 24703 heartbeater.cc:464] Registering TS with master...
06:33:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.269162 24703 heartbeater.cc:511] Master 127.23.161.254:38015 requested a full tablet report, sending...
06:33:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.270782 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24525
06:33:30.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.271020 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-0/wal/instance
06:33:30.275 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.273944 24245 ts_manager.cc:194] Registered new tserver with Master: 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.276862 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/logs
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data/info.pb
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:30.278 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.194:0
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.194
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.194
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:30.279 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
06:33:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.280968 24704 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.281131 24704 heartbeater.cc:464] Registering TS with master...
06:33:30.281 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.281596 24704 heartbeater.cc:511] Master 127.23.161.253:45177 requested a full tablet report, sending...
06:33:30.283 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.283703 24337 ts_manager.cc:194] Registered new tserver with Master: 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.285466 24702 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.285570 24702 heartbeater.cc:464] Registering TS with master...
06:33:30.285 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.285833 24702 heartbeater.cc:511] Master 127.23.161.252:42373 requested a full tablet report, sending...
06:33:30.287 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.287045 24427 ts_manager.cc:194] Registered new tserver with Master: 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:30.289 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.288847 24427 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.161.193:41763
06:33:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.426362 24512 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:30.426 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.426452 24512 catalog_manager.cc:1552] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: loading cluster ID for follower catalog manager: success
06:33:30.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.428917 24512 catalog_manager.cc:1574] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: acquiring CA information for follower catalog manager: success
06:33:30.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.430932 24512 catalog_manager.cc:1602] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:33:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.612553 24708 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:30.612 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.612854 24708 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:30.613 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.612957 24708 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.619999 24708 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.620160 24708 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
06:33:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.620201 24708 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
06:33:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.620240 24708 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
06:33:30.620 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.620306 24708 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.161.194
06:33:30.628 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.628729 24708 tablet_server_runner.cc:78] Tablet server non-default flags:
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.194:0
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data/info.pb
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.194
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.194
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/logs
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:30.629 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:30.630 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.630129 24708 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:30.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.631791 24708 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.645164 24714 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.645191 24713 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.645 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:30.645804 24716 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.646075 24708 server_base.cc:1034] running on GCE node
06:33:30.646 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.646587 24708 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:30.647 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.647388 24708 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:30.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.648599 24708 hybrid_clock.cc:648] HybridClock initialized: now 1786257210648544 us; error 35 us; skew 500 ppm
06:33:30.648 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.648892 24708 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.651191 24708 webserver.cc:460] Webserver started at http://127.23.161.194:35363/ using document root <none> and password file <none>
06:33:30.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.651847 24708 fs_manager.cc:359] Metadata directory not provided
06:33:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.651945 24708 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:30.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.652354 24708 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.654933 24708 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data/instance:
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93e8c8ec161e4104b5dc58fa5a609f99"
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.655685 24708 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal/instance:
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93e8c8ec161e4104b5dc58fa5a609f99"
06:33:30.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.660982 24708 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.008s sys 0.000s
06:33:30.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.664644 24722 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.665865 24708 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.004s sys 0.000s
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.666052 24708 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "93e8c8ec161e4104b5dc58fa5a609f99"
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:30 on dist-test-slave-1jjb"
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.666285 24708 fs_report.cc:383] FS layout report
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data/data
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:30.666 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:30.667 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:30.678 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.678416 24708 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:30.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.679380 24708 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:30.679 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.679737 24708 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:30.682 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.682751 24708 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:33:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.683002 24708 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.001s sys 0.000s
06:33:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.683174 24708 ts_tablet_manager.cc:610] Registered 0 tablets
06:33:30.683 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.683264 24708 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:33:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.742309 24708 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.194:45449
06:33:30.742 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.742347 24856 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.194:45449 every 8 connection(s)
06:33:30.743 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.743657 24708 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/data/info.pb
06:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.745020 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24708
06:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.745183 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-1/wal/instance
06:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.746354 24858 heartbeater.cc:347] Connected to a master server at 127.23.161.254:38015
06:33:30.746 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.746572 24858 heartbeater.cc:464] Registering TS with master...
06:33:30.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.748340 24858 heartbeater.cc:511] Master 127.23.161.254:38015 requested a full tablet report, sending...
06:33:30.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.750870 24245 ts_manager.cc:194] Registered new tserver with Master: 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.752873 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/logs
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data/info.pb
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) tserver
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.195:0
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.195
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.195
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:30.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:30.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true with env {}
06:33:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765141 24857 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765197 24859 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765285 24857 heartbeater.cc:464] Registering TS with master...
06:33:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765347 24859 heartbeater.cc:464] Registering TS with master...
06:33:30.765 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765637 24857 heartbeater.cc:511] Master 127.23.161.252:42373 requested a full tablet report, sending...
06:33:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.765813 24859 heartbeater.cc:511] Master 127.23.161.253:45177 requested a full tablet report, sending...
06:33:30.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.766476 24427 ts_manager.cc:194] Registered new tserver with Master: 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:30.767 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.767611 24427 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.161.194:58483
06:33:30.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:30.768641 24337 ts_manager.cc:194] Registered new tserver with Master: 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.085009 24862 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.085266 24862 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:31.085 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.085362 24862 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.092398 24862 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.092515 24862 flags.cc:425] Enabled experimental flag: --txn_keepalive_interval_ms=1000
06:33:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.092553 24862 flags.cc:425] Enabled experimental flag: --txn_staleness_tracker_interval_ms=250
06:33:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.092597 24862 flags.cc:425] Enabled experimental flag: --enable_txn_system_client_init=true
06:33:31.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.092661 24862 flags.cc:425] Enabled experimental flag: --local_ip_for_outbound_sockets=127.23.161.195
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.101477 24862 tablet_server_runner.cc:78] Tablet server non-default flags:
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:31.101 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.195:0
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data/info.pb
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.195
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=0
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_keepalive_interval_ms=1000
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_staleness_tracker_interval_ms=250
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_txn_system_client_init=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tserver_master_addrs=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --local_ip_for_outbound_sockets=127.23.161.195
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/logs
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Tablet server version:
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:31.102 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:31.103 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.103014 24862 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:31.104 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.104748 24862 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.118383 24867 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.118379 24868 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:31.118 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.118764 24870 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:31.119 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.119508 24862 server_base.cc:1034] running on GCE node
06:33:31.120 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.120036 24862 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:31.121 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.120985 24862 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.122215 24862 hybrid_clock.cc:648] HybridClock initialized: now 1786257211122165 us; error 43 us; skew 500 ppm
06:33:31.122 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.122515 24862 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:31.124 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.124583 24862 webserver.cc:460] Webserver started at http://127.23.161.195:38513/ using document root <none> and password file <none>
06:33:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.125226 24862 fs_manager.cc:359] Metadata directory not provided
06:33:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.125303 24862 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:31.125 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.125581 24862 server_base.cc:882] This appears to be a new deployment of Kudu; creating new FS layout
06:33:31.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.128320 24862 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data/instance:
06:33:31.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "371ed9133c3e433eaa40f20eb74d3876"
06:33:31.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:31 on dist-test-slave-1jjb"
06:33:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.129148 24862 fs_manager.cc:1063] Generated new instance metadata in path /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal/instance:
06:33:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "371ed9133c3e433eaa40f20eb74d3876"
06:33:31.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:31 on dist-test-slave-1jjb"
06:33:31.134 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.134284 24862 fs_manager.cc:691] Time spent creating directory manager: real 0.005s user 0.003s sys 0.002s
06:33:31.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.137540 24876 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:31.138 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.138695 24862 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.001s sys 0.000s
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.138859 24862 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "371ed9133c3e433eaa40f20eb74d3876"
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:31 on dist-test-slave-1jjb"
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.139066 24862 fs_report.cc:383] FS layout report
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data/data
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:31.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:31.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.183976 24862 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.185092 24862 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:31.185 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.185447 24862 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.188313 24862 ts_tablet_manager.cc:579] Loaded tablet metadata (0 total tablets, 0 live tablets)
06:33:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.188416 24862 ts_tablet_manager.cc:525] Time spent load tablet metadata: real 0.000s user 0.000s sys 0.000s
06:33:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.188553 24862 ts_tablet_manager.cc:610] Registered 0 tablets
06:33:31.188 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.188630 24862 ts_tablet_manager.cc:589] Time spent register tablets: real 0.000s user 0.000s sys 0.000s
06:33:31.241 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.241779 24862 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.195:41803
06:33:31.242 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.241816 25010 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.195:41803 every 8 connection(s)
06:33:31.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.243022 24862 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/data/info.pb
06:33:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.245335 25012 heartbeater.cc:347] Connected to a master server at 127.23.161.254:38015
06:33:31.245 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.245819 25012 heartbeater.cc:464] Registering TS with master...
06:33:31.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.246958 25012 heartbeater.cc:511] Master 127.23.161.254:38015 requested a full tablet report, sending...
06:33:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.248747 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 24862
06:33:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.248921 24199 external_mini_cluster.cc:1402] Reading /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/ts-2/wal/instance
06:33:31.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.248966 24245 ts_manager.cc:194] Registered new tserver with Master: 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.253353 25011 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:31.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.253499 25011 heartbeater.cc:464] Registering TS with master...
06:33:31.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.253911 25011 heartbeater.cc:511] Master 127.23.161.252:42373 requested a full tablet report, sending...
06:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.254905 24427 ts_manager.cc:194] Registered new tserver with Master: 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:31.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.255738 24427 master_service.cc:496] Signed X509 certificate for tserver {username='slave'} at 127.23.161.195:51511
06:33:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.256234 25013 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:31.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.256362 25013 heartbeater.cc:464] Registering TS with master...
06:33:31.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.256883 25013 heartbeater.cc:511] Master 127.23.161.253:45177 requested a full tablet report, sending...
06:33:31.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.258092 24337 ts_manager.cc:194] Registered new tserver with Master: 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:31.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.260586 24199 external_mini_cluster.cc:930] 3 TS(s) registered with all masters
06:33:31.262 [DEBUG - main] (MiniKuduCluster.java:195) Response:
06:33:31.262 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_masters {
}
06:33:31.263 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_masters {
masters {
id {
type: MASTER
index: 0
}
bound_rpc_address {
host: "127.23.161.254"
port: 38015
}
bound_http_address {
host: "127.23.161.254"
port: 36637
}
}
masters {
id {
type: MASTER
index: 1
}
bound_rpc_address {
host: "127.23.161.253"
port: 45177
}
bound_http_address {
host: "127.23.161.253"
port: 46215
}
}
masters {
id {
type: MASTER
index: 2
}
bound_rpc_address {
host: "127.23.161.252"
port: 42373
}
bound_http_address {
host: "127.23.161.252"
port: 37135
}
}
}
06:33:31.263 [DEBUG - main] (MiniKuduCluster.java:185) Request: get_tservers {
}
06:33:31.264 [DEBUG - main] (MiniKuduCluster.java:195) Response: get_tservers {
tservers {
id {
type: TSERVER
index: 0
}
bound_rpc_address {
host: "127.23.161.193"
port: 37599
}
bound_http_address {
host: "127.23.161.193"
port: 35807
}
}
tservers {
id {
type: TSERVER
index: 1
}
bound_rpc_address {
host: "127.23.161.194"
port: 45449
}
bound_http_address {
host: "127.23.161.194"
port: 35363
}
}
tservers {
id {
type: TSERVER
index: 2
}
bound_rpc_address {
host: "127.23.161.195"
port: 41803
}
bound_http_address {
host: "127.23.161.195"
port: 38513
}
}
}
06:33:31.264 [INFO - main] (KuduTestHarness.java:157) Creating a new Kudu client...
06:33:31.266 [DEBUG - main] (SecurityUtil.java:122) Could not login via JAAS. Using no credentials: Unable to obtain Principal Name for authentication
06:33:31.268 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:31.269 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:31.270 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:31.271 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.161.252:42373(127.23.161.252:42373)] Successfully connected to peer
06:33:31.271 [DEBUG - kudu-nio-1] (Connection.java:757) [peer master-127.23.161.254:38015(127.23.161.254:38015)] Successfully connected to peer
06:33:31.273 [DEBUG - kudu-nio-1] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.161.254
06:33:31.273 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.161.252
06:33:31.279 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.161.253:45177(127.23.161.253:45177)] Successfully connected to peer
06:33:31.286 [DEBUG - kudu-nio-0] (Negotiator.java:529) SASL mechanism PLAIN chosen for peer 127.23.161.253
06:33:31.291 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.291679 24702 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:31.293 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x909a21d6, L:/127.0.0.1:50300 - R:/127.23.161.254:38015] using SASL/PLAIN
06:33:31.297 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x80af7c06, L:/127.0.0.1:33930 - R:/127.23.161.252:42373] using SASL/PLAIN
06:33:31.299 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x1110b622, L:/127.0.0.1:57276 - R:/127.23.161.253:45177] using SASL/PLAIN
06:33:31.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:33:31.300 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=CreateTable, tablet=null, attempt=1, TimeoutTracker(timeout=50000, elapsed=33), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [28ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [32ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, deferred=null) after lookup
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.302883 24427 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:33930:
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_manager_fallback_rolling"
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column1_i"
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column2_i"
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT32
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column3_s"
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: true
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: DICT_ENCODING
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: LZ4
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 4096
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "column4_b"
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: BOOL
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_auto_incrementing: false
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) hash_schema {
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "key"
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_buckets: 2
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) seed: 0
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.305 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.305243 24427 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table txn_manager_fallback_rolling in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:33:31.326 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8dc94f51511443ff887786bd502f6c74
06:33:31.354 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.353111 24623 tablet_service.cc:1467] Processing CreateTablet for tablet 22852a0334cf48fbb5c4242fe8272a67 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
06:33:31.355 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.354969 24623 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 22852a0334cf48fbb5c4242fe8272a67. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.356 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.356264 24624 tablet_service.cc:1467] Processing CreateTablet for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
06:33:31.357 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.357153 24624 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b35c262ebf4dd2b9e78d6abc3dea00. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.373361 24782 tablet_service.cc:1467] Processing CreateTablet for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
06:33:31.374 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.374756 24782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b35c262ebf4dd2b9e78d6abc3dea00. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.375 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.375469 24930 tablet_service.cc:1467] Processing CreateTablet for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 0, RANGE (key) PARTITION UNBOUNDED
06:33:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.376735 24930 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 11b35c262ebf4dd2b9e78d6abc3dea00. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.376742 24780 tablet_service.cc:1467] Processing CreateTablet for tablet 22852a0334cf48fbb5c4242fe8272a67 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
06:33:31.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.377187 24780 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 22852a0334cf48fbb5c4242fe8272a67. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.380 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.380591 24929 tablet_service.cc:1467] Processing CreateTablet for tablet 22852a0334cf48fbb5c4242fe8272a67 (DEFAULT_TABLE table=txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]), partition=HASH (key) PARTITION 1, RANGE (key) PARTITION UNBOUNDED
06:33:31.381 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.381023 24929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet 22852a0334cf48fbb5c4242fe8272a67. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.388 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.388542 25041 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: Bootstrap starting.
06:33:31.396 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.396021 25041 tablet_bootstrap.cc:654] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: Neither blocks nor log segments found. Creating new log.
06:33:31.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.399430 25040 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Bootstrap starting.
06:33:31.401 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.400890 25041 log.cc:824] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: Log is configured to *not* fsync() on all Append() calls
06:33:31.403 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.403723 25040 tablet_bootstrap.cc:654] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Neither blocks nor log segments found. Creating new log.
06:33:31.405 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.405170 25040 log.cc:824] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Log is configured to *not* fsync() on all Append() calls
06:33:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.405912 25041 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: No bootstrap required, opened a new log
06:33:31.406 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.406286 25041 ts_tablet_manager.cc:1397] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent bootstrapping tablet: real 0.018s user 0.010s sys 0.000s
06:33:31.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.409489 25042 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: Bootstrap starting.
06:33:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.414423 25040 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: No bootstrap required, opened a new log
06:33:31.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.414682 25040 ts_tablet_manager.cc:1397] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Time spent bootstrapping tablet: real 0.016s user 0.006s sys 0.006s
06:33:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.415431 25042 tablet_bootstrap.cc:654] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: Neither blocks nor log segments found. Creating new log.
06:33:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.415169 25041 raft_consensus.cc:348] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.415601 25041 raft_consensus.cc:374] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.415699 25041 raft_consensus.cc:729] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93e8c8ec161e4104b5dc58fa5a609f99, State: Initialized, Role: FOLLOWER
06:33:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.416364 25041 consensus_queue.cc:260] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.416 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.416651 25042 log.cc:824] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: Log is configured to *not* fsync() on all Append() calls
06:33:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.418063 25041 ts_tablet_manager.cc:1428] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent starting tablet: real 0.012s user 0.007s sys 0.003s
06:33:31.418 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.418511 24857 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:31.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.422106 25041 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: Bootstrap starting.
06:33:31.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.423702 25041 tablet_bootstrap.cc:654] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: Neither blocks nor log segments found. Creating new log.
06:33:31.427 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.427241 25040 raft_consensus.cc:348] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.427891 25040 raft_consensus.cc:374] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.428 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.428025 25040 raft_consensus.cc:729] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56f0ce12c25a4723a04412921d35ce43, State: Initialized, Role: FOLLOWER
06:33:31.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.428871 25040 consensus_queue.cc:260] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.431 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.431731 25041 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: No bootstrap required, opened a new log
06:33:31.432 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.431986 25041 ts_tablet_manager.cc:1397] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent bootstrapping tablet: real 0.010s user 0.005s sys 0.001s
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433073 25042 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: No bootstrap required, opened a new log
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.432956 25041 raft_consensus.cc:348] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433323 25041 raft_consensus.cc:374] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433387 25042 ts_tablet_manager.cc:1397] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: Time spent bootstrapping tablet: real 0.024s user 0.011s sys 0.005s
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433408 25040 ts_tablet_manager.cc:1428] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Time spent starting tablet: real 0.019s user 0.015s sys 0.004s
06:33:31.433 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433640 25041 raft_consensus.cc:729] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93e8c8ec161e4104b5dc58fa5a609f99, State: Initialized, Role: FOLLOWER
06:33:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.433995 25041 consensus_queue.cc:260] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.434 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.434290 25040 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43: Bootstrap starting.
06:33:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.435704 25041 ts_tablet_manager.cc:1428] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent starting tablet: real 0.003s user 0.002s sys 0.000s
06:33:31.435 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.435884 25040 tablet_bootstrap.cc:654] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43: Neither blocks nor log segments found. Creating new log.
06:33:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.441704 25040 tablet_bootstrap.cc:492] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43: No bootstrap required, opened a new log
06:33:31.441 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.441802 25040 ts_tablet_manager.cc:1397] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43: Time spent bootstrapping tablet: real 0.008s user 0.003s sys 0.003s
06:33:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.442399 25040 raft_consensus.cc:348] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.442555 25040 raft_consensus.cc:374] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.442611 25040 raft_consensus.cc:729] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56f0ce12c25a4723a04412921d35ce43, State: Initialized, Role: FOLLOWER
06:33:31.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.442824 25040 consensus_queue.cc:260] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.444 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.443341 25040 ts_tablet_manager.cc:1428] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43: Time spent starting tablet: real 0.001s user 0.000s sys 0.000s
06:33:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.447836 25042 raft_consensus.cc:348] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.448187 25042 raft_consensus.cc:374] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.448 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.448285 25042 raft_consensus.cc:729] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 371ed9133c3e433eaa40f20eb74d3876, State: Initialized, Role: FOLLOWER
06:33:31.449 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.448834 25042 consensus_queue.cc:260] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.449864 25011 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.450452 25042 ts_tablet_manager.cc:1428] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876: Time spent starting tablet: real 0.017s user 0.008s sys 0.003s
06:33:31.451 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.450807 25042 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: Bootstrap starting.
06:33:31.452 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.452692 25042 tablet_bootstrap.cc:654] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: Neither blocks nor log segments found. Creating new log.
06:33:31.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.456306 25048 raft_consensus.cc:482] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:31.456 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.456568 25048 raft_consensus.cc:504] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.457494 25045 raft_consensus.cc:482] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.457823 25045 raft_consensus.cc:504] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.458189 25048 leader_election.cc:288] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803), 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:31.461 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.460054 25045 leader_election.cc:288] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803), 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.476470 25042 tablet_bootstrap.cc:492] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: No bootstrap required, opened a new log
06:33:31.476 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.476802 25042 ts_tablet_manager.cc:1397] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: Time spent bootstrapping tablet: real 0.026s user 0.008s sys 0.017s
06:33:31.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.477666 25042 raft_consensus.cc:348] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.477890 25042 raft_consensus.cc:374] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.477993 25042 raft_consensus.cc:729] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 371ed9133c3e433eaa40f20eb74d3876, State: Initialized, Role: FOLLOWER
06:33:31.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.478205 25042 consensus_queue.cc:260] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.478806 25042 ts_tablet_manager.cc:1428] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: Time spent starting tablet: real 0.002s user 0.000s sys 0.000s
06:33:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.491001 24962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "371ed9133c3e433eaa40f20eb74d3876" is_pre_election: true
06:33:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.491299 24649 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43" is_pre_election: true
06:33:31.491 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.491669 24962 raft_consensus.cc:2454] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 0.
06:33:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.491809 24649 raft_consensus.cc:2454] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 0.
06:33:31.492 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.492437 24727 leader_election.cc:302] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 371ed9133c3e433eaa40f20eb74d3876, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.493028 25045 raft_consensus.cc:2781] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:33:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.493192 25045 raft_consensus.cc:482] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:33:31.493 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.493286 25045 raft_consensus.cc:3037] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.495205 25060 raft_consensus.cc:482] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:31.495 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.495514 25060 raft_consensus.cc:504] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.496 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.496219 25045 raft_consensus.cc:504] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.497 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.496991 25060 leader_election.cc:288] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599), 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.497280 24962 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "371ed9133c3e433eaa40f20eb74d3876"
06:33:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.497390 24649 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "22852a0334cf48fbb5c4242fe8272a67" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43" is_pre_election: true
06:33:31.500 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.497485 24962 raft_consensus.cc:3037] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.497599 24649 raft_consensus.cc:2454] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 0.
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.497989 24725 leader_election.cc:302] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56f0ce12c25a4723a04412921d35ce43, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.498256 25060 raft_consensus.cc:2781] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.498360 25060 raft_consensus.cc:482] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.498432 25060 raft_consensus.cc:3037] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.499125 25014 tablet.cc:2369] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:33:31.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.501230 25060 raft_consensus.cc:504] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502090 24649 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "22852a0334cf48fbb5c4242fe8272a67" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502313 24649 raft_consensus.cc:3037] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502187 24961 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "22852a0334cf48fbb5c4242fe8272a67" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "371ed9133c3e433eaa40f20eb74d3876" is_pre_election: true
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502547 24961 raft_consensus.cc:2454] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 0.
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502856 24962 raft_consensus.cc:2454] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 1.
06:33:31.503 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.503033 24809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "56f0ce12c25a4723a04412921d35ce43" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" is_pre_election: true
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.503437 24727 leader_election.cc:302] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 371ed9133c3e433eaa40f20eb74d3876, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.503540 24809 raft_consensus.cc:2379] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 56f0ce12c25a4723a04412921d35ce43 in current term 1: Already voted for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in this term.
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.502411 24960 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "56f0ce12c25a4723a04412921d35ce43" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "371ed9133c3e433eaa40f20eb74d3876" is_pre_election: true
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.503880 24960 raft_consensus.cc:2379] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate 56f0ce12c25a4723a04412921d35ce43 in current term 1: Already voted for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in this term.
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.504026 24648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "11b35c262ebf4dd2b9e78d6abc3dea00" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:31.504 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.504310 24648 raft_consensus.cc:3037] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.505 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.505671 24649 raft_consensus.cc:2454] T 22852a0334cf48fbb5c4242fe8272a67 P 56f0ce12c25a4723a04412921d35ce43 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 1.
06:33:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506086 25060 leader_election.cc:288] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 election: Requested vote from peers 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599), 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506292 25060 raft_consensus.cc:2781] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Leader election won for term 1
06:33:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506445 25060 raft_consensus.cc:686] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 LEADER]: Becoming Leader. State: Replica: 93e8c8ec161e4104b5dc58fa5a609f99, State: Running, Role: LEADER
06:33:31.506 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506477 24648 raft_consensus.cc:2454] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 1.
06:33:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506932 24725 leader_election.cc:302] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 56f0ce12c25a4723a04412921d35ce43, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.506948 24563 leader_election.cc:302] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: 56f0ce12c25a4723a04412921d35ce43; no voters: 371ed9133c3e433eaa40f20eb74d3876, 93e8c8ec161e4104b5dc58fa5a609f99
06:33:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.507297 25048 raft_consensus.cc:2726] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43 [term 1 FOLLOWER]: Leader pre-election lost for term 1. Reason: could not achieve majority
06:33:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.507500 25045 leader_election.cc:288] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [CANDIDATE]: Term 1 election: Requested vote from peers 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803), 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:31.507 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.507541 25060 consensus_queue.cc:237] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } }
06:33:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.508169 25062 raft_consensus.cc:2781] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Leader election won for term 1
06:33:31.508 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.508453 25062 raft_consensus.cc:686] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 LEADER]: Becoming Leader. State: Replica: 93e8c8ec161e4104b5dc58fa5a609f99, State: Running, Role: LEADER
06:33:31.509 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.508930 25062 consensus_queue.cc:237] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.511191 24960 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "22852a0334cf48fbb5c4242fe8272a67" candidate_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "371ed9133c3e433eaa40f20eb74d3876"
06:33:31.511 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.511415 24960 raft_consensus.cc:3037] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.514603 24860 tablet.cc:2369] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:33:31.515 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.515139 24960 raft_consensus.cc:2454] T 22852a0334cf48fbb5c4242fe8272a67 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 93e8c8ec161e4104b5dc58fa5a609f99 in term 1.
06:33:31.517 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.516528 24425 catalog_manager.cc:5495] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194). New cstate: current_term: 1 leader_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } health_report { overall_health: UNKNOWN } } }
06:33:31.525 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.524803 24426 catalog_manager.cc:5495] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 reported cstate change: term changed from 0 to 1, leader changed from <none> to 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194). New cstate: current_term: 1 leader_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } health_report { overall_health: HEALTHY } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } health_report { overall_health: UNKNOWN } } }
06:33:31.535 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.535764 24707 tablet.cc:2369] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 56f0ce12c25a4723a04412921d35ce43: Can't schedule compaction. Clean time has not been advanced past its initial value.
06:33:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.631244 24426 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40248:
06:33:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
06:33:31.631 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:31.632 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
06:33:31.633 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.632601 24426 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.644236 24624 tablet_service.cc:1467] Processing CreateTablet for tablet d636f709eafc42c19dfac7bd06c74d05 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.644626 24782 tablet_service.cc:1467] Processing CreateTablet for tablet d636f709eafc42c19dfac7bd06c74d05 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.645211 24624 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d636f709eafc42c19dfac7bd06c74d05. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.645349 24782 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d636f709eafc42c19dfac7bd06c74d05. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.648487 24929 tablet_service.cc:1467] Processing CreateTablet for tablet d636f709eafc42c19dfac7bd06c74d05 (TXN_STATUS_TABLE table=kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]), partition=RANGE (txn_id) PARTITION 0 <= VALUES < 1000000
06:33:31.649 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.649161 24929 data_dirs.cc:395] Could only allocate 1 dirs of requested 3 for tablet d636f709eafc42c19dfac7bd06c74d05. 1 dirs total, 0 dirs full, 0 dirs failed
06:33:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.651340 25040 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: Bootstrap starting.
06:33:31.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.652302 25041 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: Bootstrap starting.
06:33:31.653 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.653038 25040 tablet_bootstrap.cc:654] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: Neither blocks nor log segments found. Creating new log.
06:33:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.653925 25041 tablet_bootstrap.cc:654] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: Neither blocks nor log segments found. Creating new log.
06:33:31.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.654711 25040 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: No bootstrap required, opened a new log
06:33:31.655 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.654874 25040 ts_tablet_manager.cc:1397] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: Time spent bootstrapping tablet: real 0.004s user 0.000s sys 0.003s
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.655537 25040 raft_consensus.cc:348] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.655781 25040 raft_consensus.cc:374] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.655856 25040 raft_consensus.cc:729] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 56f0ce12c25a4723a04412921d35ce43, State: Initialized, Role: FOLLOWER
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.655995 25041 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: No bootstrap required, opened a new log
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656047 25042 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: Bootstrap starting.
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656158 25041 ts_tablet_manager.cc:1397] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656126 25040 consensus_queue.cc:260] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656551 25048 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656802 25048 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.656826 25041 raft_consensus.cc:348] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.657050 25041 raft_consensus.cc:374] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.657110 25041 raft_consensus.cc:729] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 93e8c8ec161e4104b5dc58fa5a609f99, State: Initialized, Role: FOLLOWER
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.657341 25040 ts_tablet_manager.cc:1428] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: Time spent starting tablet: real 0.002s user 0.000s sys 0.002s
06:33:31.657 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.657666 25042 tablet_bootstrap.cc:654] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: Neither blocks nor log segments found. Creating new log.
06:33:31.658 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.657898 25041 consensus_queue.cc:260] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.659487 25041 ts_tablet_manager.cc:1428] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: Time spent starting tablet: real 0.003s user 0.001s sys 0.000s
06:33:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.659595 25042 tablet_bootstrap.cc:492] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: No bootstrap required, opened a new log
06:33:31.660 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.659744 25042 ts_tablet_manager.cc:1397] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: Time spent bootstrapping tablet: real 0.004s user 0.003s sys 0.000s
06:33:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.660849 25060 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.661 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.661157 25060 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.661998 25042 raft_consensus.cc:348] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.662330 25042 raft_consensus.cc:374] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Consensus starting up: Expiring failure detector timer to make a prompt election more likely
06:33:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.662416 25042 raft_consensus.cc:729] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Becoming Follower/Learner. State: Replica: 371ed9133c3e433eaa40f20eb74d3876, State: Initialized, Role: FOLLOWER
06:33:31.662 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.662683 25042 consensus_queue.cc:260] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.663360 25042 ts_tablet_manager.cc:1428] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: Time spent starting tablet: real 0.004s user 0.002s sys 0.000s
06:33:31.663 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.663440 25050 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: TxnStatusTablet state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 0 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.664 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.663829 25050 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.739 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.739670 25050 raft_consensus.cc:482] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:31.740 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.739857 25050 raft_consensus.cc:504] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.741 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.741173 25050 leader_election.cc:288] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [CANDIDATE]: Term 1 pre-election: Requested pre-vote from peers 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599), 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:31.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.748592 24809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d636f709eafc42c19dfac7bd06c74d05" candidate_uuid: "371ed9133c3e433eaa40f20eb74d3876" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" is_pre_election: true
06:33:31.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.748601 24648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d636f709eafc42c19dfac7bd06c74d05" candidate_uuid: "371ed9133c3e433eaa40f20eb74d3876" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43" is_pre_election: true
06:33:31.748 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.748869 24648 raft_consensus.cc:2454] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 371ed9133c3e433eaa40f20eb74d3876 in term 0.
06:33:31.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.748845 24809 raft_consensus.cc:2454] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate 371ed9133c3e433eaa40f20eb74d3876 in term 0.
06:33:31.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.749478 24880 leader_election.cc:302] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [CANDIDATE]: Term 1 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 3 yes votes; 0 no votes. yes voters: 371ed9133c3e433eaa40f20eb74d3876, 56f0ce12c25a4723a04412921d35ce43, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.749894 25050 raft_consensus.cc:2781] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Leader pre-election won for term 1
06:33:31.752 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.750025 25050 raft_consensus.cc:482] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Starting leader election (no leader contacted us within the election timeout)
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.750164 25050 raft_consensus.cc:3037] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.752213 25050 raft_consensus.cc:504] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.752753 25050 leader_election.cc:288] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [CANDIDATE]: Term 1 election: Requested vote from peers 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599), 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.753091 24648 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d636f709eafc42c19dfac7bd06c74d05" candidate_uuid: "371ed9133c3e433eaa40f20eb74d3876" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "56f0ce12c25a4723a04412921d35ce43"
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.753299 24648 raft_consensus.cc:3037] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.753222 24809 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "d636f709eafc42c19dfac7bd06c74d05" candidate_uuid: "371ed9133c3e433eaa40f20eb74d3876" candidate_term: 1 candidate_status { last_received { term: 0 index: 0 } } ignore_live_leader: false dest_uuid: "93e8c8ec161e4104b5dc58fa5a609f99"
06:33:31.753 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.753430 24809 raft_consensus.cc:3037] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 0 FOLLOWER]: Advancing to term 1
06:33:31.755 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.755501 24809 raft_consensus.cc:2454] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 371ed9133c3e433eaa40f20eb74d3876 in term 1.
06:33:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.755925 24880 leader_election.cc:302] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [CANDIDATE]: Term 1 election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: 371ed9133c3e433eaa40f20eb74d3876, 93e8c8ec161e4104b5dc58fa5a609f99; no voters:
06:33:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.755975 24648 raft_consensus.cc:2454] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 1 FOLLOWER]: Leader election vote request: Granting yes vote for candidate 371ed9133c3e433eaa40f20eb74d3876 in term 1.
06:33:31.756 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.756265 25050 raft_consensus.cc:2781] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 FOLLOWER]: Leader election won for term 1
06:33:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.756807 25050 raft_consensus.cc:686] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [term 1 LEADER]: Becoming Leader. State: Replica: 371ed9133c3e433eaa40f20eb74d3876, State: Running, Role: LEADER
06:33:31.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.757613 25050 consensus_queue.cc:237] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 0, Last appended: 0.0, Last appended by leader: 0, Current term: 1, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } }
06:33:31.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.762033 25078 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: TxnStatusTablet state changed. Reason: New leader 371ed9133c3e433eaa40f20eb74d3876. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.762315 25078 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: This TxnStatusTablet replica's current role is: LEADER
06:33:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.763624 25080 txn_status_manager.cc:873] Waiting until node catch up with all replicated operations in previous term...
06:33:31.763 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.763886 25080 txn_status_manager.cc:929] Loading transaction status metadata into memory...
06:33:31.764 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.764173 24426 catalog_manager.cc:5495] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 reported cstate change: term changed from 0 to 1, leader changed from <none> to 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195). New cstate: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } health_report { overall_health: UNKNOWN } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } health_report { overall_health: HEALTHY } } }
06:33:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.792116 24809 raft_consensus.cc:1261] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99 [term 1 FOLLOWER]: Refusing update from remote peer 371ed9133c3e433eaa40f20eb74d3876: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:33:31.792 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.792186 24648 raft_consensus.cc:1261] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43 [term 1 FOLLOWER]: Refusing update from remote peer 371ed9133c3e433eaa40f20eb74d3876: Log matching property violated. Preceding OpId in replica: term: 0 index: 0. Preceding OpId from leader: term: 1 index: 2. (index mismatch)
06:33:31.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.793079 25078 consensus_queue.cc:1035] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [LEADER]: Connected to new peer: Peer: permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:31.793 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.793437 25050 consensus_queue.cc:1035] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.799908 25060 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: TxnStatusTablet state changed. Reason: New leader 371ed9133c3e433eaa40f20eb74d3876. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.800 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.800277 25060 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.806001 25060 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.806316 25060 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 93e8c8ec161e4104b5dc58fa5a609f99: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.809697 25048 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: TxnStatusTablet state changed. Reason: New leader 371ed9133c3e433eaa40f20eb74d3876. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.811 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.809976 25048 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.812 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.812402 25083 mvcc.cc:205] Tried to move back new op lower bound from 7316509539490906112 to 7316509539362914304. Current Snapshot: MvccSnapshot[applied={T|T < 7316509539490906112}]
06:33:31.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.815958 25050 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.816 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.816256 25050 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: This TxnStatusTablet replica's current role is: LEADER
06:33:31.818 [DEBUG - Time-limited test] (KuduTransaction.java:702) starting keepalive heartbeating with period 500 ms (txn ID 0)
06:33:31.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.818106 25078 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: TxnStatusTablet state changed. Reason: Peer health change. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.818 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.818429 25078 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 371ed9133c3e433eaa40f20eb74d3876: This TxnStatusTablet replica's current role is: LEADER
06:33:31.824 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:894) Opened table 8dc94f51511443ff887786bd502f6c74
06:33:31.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.824252 25048 tablet_replica.cc:439] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: TxnStatusTablet state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 1 leader_uuid: "371ed9133c3e433eaa40f20eb74d3876" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 } } peers { permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" member_type: VOTER last_known_addr { host: "127.23.161.194" port: 45449 } } peers { permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 } } }
06:33:31.824 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.161.252:42373
06:33:31.824 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.824483 25048 tablet_replica.cc:442] T d636f709eafc42c19dfac7bd06c74d05 P 56f0ce12c25a4723a04412921d35ce43: This TxnStatusTablet replica's current role is: FOLLOWER
06:33:31.825 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 2
}
}
06:33:31.825 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.825389 25086 mvcc.cc:205] Tried to move back new op lower bound from 7316509539490906112 to 7316509539362914304. Current Snapshot: MvccSnapshot[applied={T|T < 7316509539490906112 or (T in {7316509539490906112})}]
06:33:31.826 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.826365 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24393
06:33:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.826030 25084 mvcc.cc:205] Tried to move back new op lower bound from 7316509539490906112 to 7316509539362914304. Current Snapshot: MvccSnapshot[applied={T|T < 7316509539490906112}]
06:33:31.827 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.827695 24470 generic_service.cc:196] Checking for leaks (request via RPC)
06:33:31.946 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.946415 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24393
06:33:31.978 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.978248 25060 consensus_queue.cc:1035] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:31.981 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.161.252:42373(127.23.161.252:42373)] handling channelInactive
06:33:31.981 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state READY due to: connection closed
06:33:31.983 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:33:31.986 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.985849 25011 heartbeater.cc:644] Failed to heartbeat to 127.23.161.252:42373 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:31.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.988722 25061 consensus_queue.cc:1035] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:31.995 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.995458 24857 heartbeater.cc:644] Failed to heartbeat to 127.23.161.252:42373 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:31.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:31.996538 24702 heartbeater.cc:644] Failed to heartbeat to 127.23.161.252:42373 (0 consecutive failures): Network error: Failed to send heartbeat to master: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:31.999 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:31.998692 25061 consensus_queue.cc:1035] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" member_type: VOTER last_known_addr { host: "127.23.161.193" port: 37599 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:32.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.005383 25060 consensus_queue.cc:1035] T 11b35c262ebf4dd2b9e78d6abc3dea00 P 93e8c8ec161e4104b5dc58fa5a609f99 [LEADER]: Connected to new peer: Peer: permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" member_type: VOTER last_known_addr { host: "127.23.161.195" port: 41803 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 1, Last known committed idx: 0, Time since last communication: 0.000s
06:33:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.245906 25101 raft_consensus.cc:482] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.246187 25101 raft_consensus.cc:504] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:32.246 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.246645 25102 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.246829 25102 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:32.247 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.247509 25102 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.248064 24255 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb210d252711436ab28b2ae64acb48a3" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "ceef4bfa72a4468aad391f4034246263" is_pre_election: true
06:33:32.248 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.248345 24255 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate eb210d252711436ab28b2ae64acb48a3 in term 1.
06:33:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.248895 24319 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate won. Election summary: received 2 responses out of 3 voters: 2 yes votes; 0 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, eb210d252711436ab28b2ae64acb48a3; no voters:
06:33:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.248948 25101 leader_election.cc:288] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 2 pre-election: Requested pre-vote from peers eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.249159 25102 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Leader pre-election won for term 2
06:33:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.249279 25102 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Starting leader election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:32.249 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.249351 25102 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 1 FOLLOWER]: Advancing to term 2
06:33:32.251 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.251008 24346 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ceef4bfa72a4468aad391f4034246263" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "eb210d252711436ab28b2ae64acb48a3" is_pre_election: true
06:33:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.252123 24321 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:32.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.252408 24232 leader_election.cc:334] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 2 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:32.253 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.253037 25102 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.253530 24346 raft_consensus.cc:2379] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Leader pre-election vote request: Denying vote to candidate ceef4bfa72a4468aad391f4034246263 in current term 2: Already voted for candidate eb210d252711436ab28b2ae64acb48a3 in this term.
06:33:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.253674 25102 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 election: Requested vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.254168 24255 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb210d252711436ab28b2ae64acb48a3" candidate_term: 2 candidate_status { last_received { term: 1 index: 11 } } ignore_live_leader: false dest_uuid: "ceef4bfa72a4468aad391f4034246263"
06:33:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.254369 24232 leader_election.cc:302] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 2 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: ceef4bfa72a4468aad391f4034246263; no voters: d573f1a12d4c4086bbf20f8e73481ad7, eb210d252711436ab28b2ae64acb48a3
06:33:32.254 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.254419 24255 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 1 FOLLOWER]: Advancing to term 2
06:33:32.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.255534 24321 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:32.256 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.256738 24255 raft_consensus.cc:2454] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Leader election vote request: Granting yes vote for candidate eb210d252711436ab28b2ae64acb48a3 in term 2.
06:33:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.256916 25101 raft_consensus.cc:2726] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Leader pre-election lost for term 2. Reason: could not achieve majority
06:33:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.257222 24319 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 2 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, eb210d252711436ab28b2ae64acb48a3; no voters: d573f1a12d4c4086bbf20f8e73481ad7
06:33:32.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.257510 25102 raft_consensus.cc:2781] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Leader election won for term 2
06:33:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.257916 25102 raft_consensus.cc:686] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 LEADER]: Becoming Leader. State: Replica: eb210d252711436ab28b2ae64acb48a3, State: Running, Role: LEADER
06:33:32.258 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.258337 25102 consensus_queue.cc:237] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 11, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 2, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.259760 25108 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb210d252711436ab28b2ae64acb48a3. Latest consensus state: current_term: 2 leader_uuid: "eb210d252711436ab28b2ae64acb48a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:32.260 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.260156 25108 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: LEADER
06:33:32.261 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.260932 25110 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:33:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.264010 25110 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]
06:33:32.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.264796 25110 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]
06:33:32.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.266759 25110 tablet_loader.cc:96] loaded metadata for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.267206 25110 tablet_loader.cc:96] loaded metadata for tablet 22852a0334cf48fbb5c4242fe8272a67 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.267582 25110 tablet_loader.cc:96] loaded metadata for tablet d636f709eafc42c19dfac7bd06c74d05 (table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6])
06:33:32.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.267868 25110 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:33:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.268915 25110 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:32.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.269014 25110 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:33:32.270 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.270493 25110 catalog_manager.cc:1504] Loading token signing keys...
06:33:32.271 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.271572 25110 catalog_manager.cc:5879] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Loaded TSK: 0
06:33:32.272 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.272619 25110 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:33:32.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:32.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.329 [INFO - kudu-nio-1] (AsyncKuduClient.java:2355) Invalidating location master-127.23.161.252:42373(127.23.161.252:42373) for tablet Kudu Master: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.336987 24255 raft_consensus.cc:1261] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Refusing update from remote peer eb210d252711436ab28b2ae64acb48a3: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 2 index: 12. (index mismatch)
06:33:32.337 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.337591 25108 consensus_queue.cc:1035] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [LEADER]: Connected to new peer: Peer: permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 12, Last known committed idx: 11, Time since last communication: 0.000s
06:33:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.339066 25101 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: New leader eb210d252711436ab28b2ae64acb48a3. Latest consensus state: current_term: 2 leader_uuid: "eb210d252711436ab28b2ae64acb48a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:32.339 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.339371 25101 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: FOLLOWER
06:33:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.340727 25108 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 2 leader_uuid: "eb210d252711436ab28b2ae64acb48a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.341099 25108 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: LEADER
06:33:32.341 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.341663 25101 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 2 leader_uuid: "eb210d252711436ab28b2ae64acb48a3" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.341881 25101 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: FOLLOWER
06:33:32.342 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.342288 24321 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 -> Peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Couldn't send request to peer d573f1a12d4c4086bbf20f8e73481ad7. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:33:32.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:32.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:32.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:33:32.352 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=2, TimeoutTracker(timeout=250, elapsed=23), Traces: [0ms] sending RPC to server master-127.23.161.252:42373, [1ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] delaying RPC due to: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=Deferred@1286321055(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1004485492@53f1ec50, errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/92133007@ac17f12)) after lookup
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.392795 24336 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:35288:
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:33:32.399 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
06:33:32.400 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.400427 24336 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:33:32.518 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:32.517202 24859 heartbeater.cc:503] Master 127.23.161.253:45177 was elected leader, sending a full tablet report...
06:33:32.887 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:32.886849 24321 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 -> Peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Couldn't send request to peer d573f1a12d4c4086bbf20f8e73481ad7. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:33:33.436 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:33.436312 24321 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 -> Peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Couldn't send request to peer d573f1a12d4c4086bbf20f8e73481ad7. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
06:33:33.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:33.951609 24321 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111) [suppressed 21 similar messages]
06:33:33.952 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:33.952574 24321 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 -> Peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Couldn't send request to peer d573f1a12d4c4086bbf20f8e73481ad7. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
06:33:33.988 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 11b35c262ebf4dd2b9e78d6abc3dea00 for table 'txn_manager_fallback_rolling' with partition [0x00000000, 0x00000001)
06:33:33.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet 22852a0334cf48fbb5c4242fe8272a67 for table 'txn_manager_fallback_rolling' with partition [0x00000001, 0x00000002)
06:33:33.989 [DEBUG - kudu-nio-0] (TableLocationsCache.java:192) Discovered table locations: [Tablet{lowerBoundPartitionKey=0x00000000, upperBoundPartitionKey=0x00000001, ttl=299999, tablet=11b35c262ebf4dd2b9e78d6abc3dea00@[371ed9133c3e433eaa40f20eb74d3876(127.23.161.195:41803),56f0ce12c25a4723a04412921d35ce43(127.23.161.193:37599),93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)[L]]}, Tablet{lowerBoundPartitionKey=0x00000001, upperBoundPartitionKey=0x00000002, ttl=299999, tablet=22852a0334cf48fbb5c4242fe8272a67@[371ed9133c3e433eaa40f20eb74d3876(127.23.161.195:41803),56f0ce12c25a4723a04412921d35ce43(127.23.161.193:37599),93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)[L]]}, NonCoveredRange{lowerBoundPartitionKey=0x00000002, upperBoundPartitionKey=0x, ttl=299999}]
06:33:33.989 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=Write, tablet=null, attempt=1, sequence_id=-1, TimeoutTracker(timeout=30000, elapsed=2), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC GetTableLocations: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC GetTableLocations: received response from server master-127.23.161.253:45177: OK, deferred=null) after lookup
06:33:33.990 [DEBUG - kudu-nio-0] (Connection.java:748) [peer 93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)] connecting to peer
06:33:33.991 [DEBUG - kudu-nio-1] (Connection.java:757) [peer 93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)] Successfully connected to peer
06:33:34.003 [DEBUG - kudu-nio-1] (Negotiator.java:919) Authenticated connection [id: 0x74afe952, L:/127.0.0.1:40016 - R:/127.23.161.194:45449] using TOKEN/null
06:33:34.007 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.007802 24773 master_proxy_rpc.cc:203] Re-attempting GetTableSchema request to leader Master (127.23.161.252:42373)
06:33:34.010 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.010638 24704 heartbeater.cc:503] Master 127.23.161.253:45177 was elected leader, sending a full tablet report...
06:33:34.014 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.014209 25013 heartbeater.cc:503] Master 127.23.161.253:45177 was elected leader, sending a full tablet report...
06:33:34.044 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.161.252:42373
06:33:34.045 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 2
}
}
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.046243 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/logs
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:34.046 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.252:42373
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.252
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37135
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:34.047 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
06:33:34.048 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
06:33:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.376881 25151 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.377189 25151 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:34.377 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.377300 25151 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383481 25151 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383616 25151 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383669 25151 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383747 25151 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383797 25151 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383888 25151 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:33:34.383 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.383936 25151 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41539/jwks.json
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.392154 25151 master_runner.cc:386] Master server non-default flags:
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.252:42373
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:34.392 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.252
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=37135
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/logs
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:34.393 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:34.394 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.393715 25151 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:34.395 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.395627 25151 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.407240 25157 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.407269 25158 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:34.407 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.407302 25160 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:34.408 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.408349 25151 server_base.cc:1034] running on GCE node
06:33:34.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.409119 25151 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:34.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.410434 25151 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:34.411 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.411645 25151 hybrid_clock.cc:648] HybridClock initialized: now 1786257214411593 us; error 42 us; skew 500 ppm
06:33:34.412 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.411979 25151 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.414099 25151 webserver.cc:460] Webserver started at http://127.23.161.252:37135/ using document root <none> and password file <none>
06:33:34.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.414726 25151 fs_manager.cc:359] Metadata directory not provided
06:33:34.415 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.414914 25151 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:34.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.419502 25151 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.002s sys 0.001s
06:33:34.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.422180 25166 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:34.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.423321 25151 fs_manager.cc:725] Time spent opening block manager: real 0.002s user 0.000s sys 0.002s
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.423511 25151 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "d573f1a12d4c4086bbf20f8e73481ad7"
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:29 on dist-test-slave-1jjb"
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.424424 25151 fs_report.cc:383] FS layout report
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/wal
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/data
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:34.424 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:34.425 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:34.429 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.429158 24321 consensus_peers.cc:487] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 -> Peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Couldn't send request to peer d573f1a12d4c4086bbf20f8e73481ad7. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
06:33:34.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.438580 24212 master_runner.cc:186] Error getting master registration for 127.23.161.252:42373: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111),
06:33:34.438 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.438771 24212 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:34.442 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.442315 25151 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.443095 25151 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:34.443 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.443390 25151 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.472901 25238 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.252:42373 every 8 connection(s)
06:33:34.473 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.472909 25151 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.252:42373
06:33:34.474 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.474520 25151 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-2/data/info.pb
06:33:34.477 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.477015 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 25151
06:33:34.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:33:34.477 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.161.253:45177
06:33:34.477 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 1
}
}
06:33:34.478 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.477913 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24297
06:33:34.479 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.479079 24384 generic_service.cc:196] Checking for leaks (request via RPC)
06:33:34.481 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.481814 25239 sys_catalog.cc:263] Verifying existing consensus state
06:33:34.485 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.485273 25239 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Bootstrap starting.
06:33:34.501 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.501650 25239 log.cc:824] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Log is configured to *not* fsync() on all Append() calls
06:33:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.515805 25239 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Bootstrap replayed 1/1 log segments. Stats: ops{read=11 overwritten=0 applied=11 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:33:34.516 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.516381 25239 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Bootstrap complete.
06:33:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.536949 25239 raft_consensus.cc:348] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:34.538 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.538220 25239 raft_consensus.cc:729] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Becoming Follower/Learner. State: Replica: d573f1a12d4c4086bbf20f8e73481ad7, State: Initialized, Role: FOLLOWER
06:33:34.539 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.539000 25239 consensus_queue.cc:260] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 11, Last appended: 1.11, Last appended by leader: 11, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.539855 25246 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 1 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.540 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.540397 25246 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: FOLLOWER
06:33:34.541 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.541088 25239 sys_catalog.cc:564] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: configured and running, proceeding with master startup.
06:33:34.550 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.549247 24857 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.560882 25257 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:34.561 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.561179 25257 catalog_manager.cc:1552] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: loading cluster ID for follower catalog manager: success
06:33:34.566 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.566190 25257 catalog_manager.cc:1574] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: acquiring CA information for follower catalog manager: success
06:33:34.569 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.569310 25257 catalog_manager.cc:1602] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:33:34.636 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.636425 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24297
06:33:34.670 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.670286 24878 connection.cc:535] server connection from 127.0.0.1:59066 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:33:34.670 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.161.253:45177(127.23.161.253:45177)] handling channelInactive
06:33:34.671 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state READY due to: connection closed
06:33:34.671 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.23.161.253:45177(127.23.161.253:45177) for tablet Kudu Master: connection closed
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.671970 25245 negotiation.cc:337] Failed RPC negotiation. Trace:
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635766 (+ 0us) reactor.cc:625] Submitting negotiation task for server connection from 127.0.0.1:40744 (local address 127.23.161.252:42373)
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635949 (+ 183us) server_negotiation.cc:206] Beginning negotiation
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635955 (+ 6us) server_negotiation.cc:399] Waiting for connection header
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635972 (+ 17us) server_negotiation.cc:407] Connection header received
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.636039 (+ 67us) server_negotiation.cc:365] Received NEGOTIATE NegotiatePB request
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.636047 (+ 8us) server_negotiation.cc:461] Received NEGOTIATE request from client
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.636142 (+ 95us) server_negotiation.cc:377] Sending NEGOTIATE NegotiatePB response
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.636224 (+ 82us) server_negotiation.cc:226] Negotiated authn=CERTIFICATE
06:33:34.672 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.670526 (+ 34302us) negotiation.cc:327] Negotiation complete: Network error: Server connection negotiation failed: server connection from 127.0.0.1:40744: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"server-negotiator.queue_time_us":70}
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.672283 25259 negotiation.cc:337] Failed RPC negotiation. Trace:
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.579625 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.23.161.253:45177 (local address 127.0.0.1:35304)
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.579770 (+ 145us) negotiation.cc:107] Waiting for socket to connect
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.579786 (+ 16us) client_negotiation.cc:174] Beginning negotiation
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.579866 (+ 80us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635363 (+ 55497us) client_negotiation.cc:271] Received NEGOTIATE NegotiatePB response
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635369 (+ 6us) client_negotiation.cc:394] Received NEGOTIATE response from server
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635406 (+ 37us) client_negotiation.cc:189] Negotiated authn=CERTIFICATE
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635606 (+ 200us) client_negotiation.cc:514] Sending TLS_HANDSHAKE message to server
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.635623 (+ 17us) client_negotiation.cc:252] Sending TLS_HANDSHAKE NegotiatePB request
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:34.671389 (+ 35766us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: BlockingRecv error: recv got EOF from 127.23.161.253:45177 (error 108)
06:33:34.673 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":46}
06:33:34.675 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:33:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.672725 25151 master_runner.cc:186] Error getting master registration for 127.23.161.253:45177: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: BlockingRecv error: recv got EOF from 127.23.161.253:45177 (error 108),
06:33:34.680 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.680151 25151 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:34.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:34.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:34.689 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.161.252:42373(127.23.161.252:42373)] Successfully connected to peer
06:33:34.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.253:45177: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.703 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0xcdfa0552, L:/127.0.0.1:40748 - R:/127.23.161.252:42373] using TOKEN/null
06:33:34.705 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:34.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:34.710 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.253:45177: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.715 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:34.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:34.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.253:45177: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.731 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:34.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.734434 25263 raft_consensus.cc:482] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Starting pre-election (detected failure of leader eb210d252711436ab28b2ae64acb48a3)
06:33:34.734 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.734661 25263 raft_consensus.cc:504] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:34.735 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.735391 25263 leader_election.cc:288] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:34.736 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.736055 24232 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111) [suppressed 8 similar messages]
06:33:34.738 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.737944 24232 leader_election.cc:334] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:34.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.743865 25199 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ceef4bfa72a4468aad391f4034246263" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" is_pre_election: true
06:33:34.744 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.744619 25199 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate ceef4bfa72a4468aad391f4034246263 in term 1.
06:33:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.745231 24232 leader_election.cc:302] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7; no voters: eb210d252711436ab28b2ae64acb48a3
06:33:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.745568 25263 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Leader pre-election won for term 3
06:33:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.745698 25263 raft_consensus.cc:482] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Starting leader election (detected failure of leader eb210d252711436ab28b2ae64acb48a3)
06:33:34.745 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.745782 25263 raft_consensus.cc:3037] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 2 FOLLOWER]: Advancing to term 3
06:33:34.749 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=4, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.23.161.253:45177, [183ms] received response from server master-127.23.161.253:45177: Network error: connection closed, [183ms] delaying RPC due to: Network error: connection closed, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: Network error: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [216ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [217ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: Network error: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [227ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [227ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [227ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: Network error: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, deferred=Deferred@898599456(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1004485492@24055960 -> (continuation of Deferred@861008666 after retry RPC after error@1738333045) -> (continuation of Deferred@780895020 after retry RPC after error@541786564) -> (continuation of Deferred@473890804 after retry RPC after error@502843509), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/92133007@6f98aa05 -> (continuation of Deferred@861008666 after retry RPC after error@1738333045) -> (continuation of Deferred@780895020 after retry RPC after error@541786564) -> (continuation of Deferred@473890804 after retry RPC after error@502843509)))
06:33:34.749 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.748775 25263 raft_consensus.cc:504] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 3 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:34.749 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
06:33:34.750 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.749464 25263 leader_election.cc:288] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 election: Requested vote from peers eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.750906 25199 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "ceef4bfa72a4468aad391f4034246263" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "d573f1a12d4c4086bbf20f8e73481ad7"
06:33:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.751140 25199 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 1 FOLLOWER]: Advancing to term 3
06:33:34.751 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.751573 24232 leader_election.cc:334] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 election: RPC error from VoteRequest() call to peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111)
06:33:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.757382 25199 raft_consensus.cc:2454] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Leader election vote request: Granting yes vote for candidate ceef4bfa72a4468aad391f4034246263 in term 3.
06:33:34.757 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.757755 24232 leader_election.cc:302] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [CANDIDATE]: Term 3 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7; no voters: eb210d252711436ab28b2ae64acb48a3
06:33:34.758 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.758165 25263 raft_consensus.cc:2781] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 3 FOLLOWER]: Leader election won for term 3
06:33:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.758652 25263 raft_consensus.cc:686] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [term 3 LEADER]: Becoming Leader. State: Replica: ceef4bfa72a4468aad391f4034246263, State: Running, Role: LEADER
06:33:34.760 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.759307 25263 consensus_queue.cc:237] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 12, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 3, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:34.761 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.761339 25266 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ceef4bfa72a4468aad391f4034246263. Latest consensus state: current_term: 3 leader_uuid: "ceef4bfa72a4468aad391f4034246263" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.761757 25266 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: LEADER
06:33:34.766 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.766208 25268 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:33:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.769033 25268 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]
06:33:34.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:34.769 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.769868 25268 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]
06:33:34.771 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.771 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.253:45177: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.772 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.772547 25268 tablet_loader.cc:96] loaded metadata for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.773049 25268 tablet_loader.cc:96] loaded metadata for tablet 22852a0334cf48fbb5c4242fe8272a67 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.773469 25268 tablet_loader.cc:96] loaded metadata for tablet d636f709eafc42c19dfac7bd06c74d05 (table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6])
06:33:34.773 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.773774 25268 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:33:34.774 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.774967 25268 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:34.775 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.775066 25268 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:33:34.776 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.776314 25268 catalog_manager.cc:1504] Loading token signing keys...
06:33:34.777 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.777168 25268 catalog_manager.cc:5879] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263: Loaded TSK: 0
06:33:34.778 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.777978 25268 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:33:34.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:34.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.790 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.253:45177: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177
06:33:34.792 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:33:34.793 [DEBUG - kudu-nio-1] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=2, TimeoutTracker(timeout=250, elapsed=24), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: Network error: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [6ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: Network error: Failed to connect to peer master-127.23.161.253:45177(127.23.161.253:45177): Connection refused: /127.23.161.253:45177, [24ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: OK, deferred=Deferred@2127357358(state=PENDING, result=null, callback=(continuation of Deferred@1946429473 after retry RPC after error@1525880667), errback=(continuation of Deferred@1946429473 after retry RPC after error@1525880667))) after lookup
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.833657 24245 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:36430:
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:34.834 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.835 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
06:33:34.836 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.835769 24245 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:33:34.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:34.838603 24232 consensus_peers.cc:487] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 -> Peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Couldn't send request to peer eb210d252711436ab28b2ae64acb48a3. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:33:34.862 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.862349 25199 raft_consensus.cc:1261] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Refusing update from remote peer ceef4bfa72a4468aad391f4034246263: Log matching property violated. Preceding OpId in replica: term: 1 index: 11. Preceding OpId from leader: term: 3 index: 13. (index mismatch)
06:33:34.863 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.863219 25266 consensus_queue.cc:1035] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [LEADER]: Connected to new peer: Peer: permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 13, Last known committed idx: 11, Time since last communication: 0.000s
06:33:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.866776 25283 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ceef4bfa72a4468aad391f4034246263" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.867170 25283 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: FOLLOWER
06:33:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.867465 25246 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader ceef4bfa72a4468aad391f4034246263. Latest consensus state: current_term: 3 leader_uuid: "ceef4bfa72a4468aad391f4034246263" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.867 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.867745 25246 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: FOLLOWER
06:33:34.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.868853 25263 sys_catalog.cc:455] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 3 leader_uuid: "ceef4bfa72a4468aad391f4034246263" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.869184 25263 sys_catalog.cc:458] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 [sys.catalog]: This master's current role is: LEADER
06:33:34.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.869436 25246 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 3 leader_uuid: "ceef4bfa72a4468aad391f4034246263" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:34.869 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:34.869707 25246 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: FOLLOWER
06:33:35.021 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.020912 24702 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:35.023 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.023028 25011 heartbeater.cc:347] Connected to a master server at 127.23.161.252:42373
06:33:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.023936 25187 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" instance_seqno: 1786257210255485) as {username='slave'} at 127.23.161.193:43959; Asking this server to re-register.
06:33:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.024503 25184 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" instance_seqno: 1786257211234083) as {username='slave'} at 127.23.161.195:52495; Asking this server to re-register.
06:33:35.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.024675 24702 heartbeater.cc:464] Registering TS with master...
06:33:35.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.025115 25011 heartbeater.cc:464] Registering TS with master...
06:33:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.026489 25184 ts_manager.cc:194] Registered new tserver with Master: 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:35.026 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.026572 25187 ts_manager.cc:194] Registered new tserver with Master: 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:35.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:35.329228 24232 consensus_peers.cc:487] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 -> Peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Couldn't send request to peer eb210d252711436ab28b2ae64acb48a3. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:33:35.554 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.554746 25187 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" instance_seqno: 1786257210734327) as {username='slave'} at 127.23.161.194:34925; Asking this server to re-register.
06:33:35.555 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.555476 24857 heartbeater.cc:464] Registering TS with master...
06:33:35.556 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:35.556723 25187 ts_manager.cc:194] Registered new tserver with Master: 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:35.858 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:35.858520 24232 consensus_peers.cc:487] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 -> Peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Couldn't send request to peer eb210d252711436ab28b2ae64acb48a3. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
06:33:36.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:36.016557 24703 heartbeater.cc:503] Master 127.23.161.254:38015 was elected leader, sending a full tablet report...
06:33:36.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:36.017804 25012 heartbeater.cc:503] Master 127.23.161.254:38015 was elected leader, sending a full tablet report...
06:33:36.359 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:36.359609 24232 consensus_peers.cc:487] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 -> Peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Couldn't send request to peer eb210d252711436ab28b2ae64acb48a3. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111). This is attempt 16: this message will repeat every 5th retry.
06:33:36.526 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:36.526161 24858 heartbeater.cc:503] Master 127.23.161.254:38015 was elected leader, sending a full tablet report...
06:33:36.698 [INFO - Time-limited test] (MiniKuduCluster.java:380) Starting master server 127.23.161.253:45177
06:33:36.699 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: start_daemon {
id {
type: MASTER
index: 1
}
}
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:36.699880 24199 external_mini_cluster.cc:1326] Running /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --block_manager=log
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=localhost
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/logs
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) master
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) run
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.253:45177
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.253
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46215
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled
06:33:36.700 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100 with env {}
06:33:36.868 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:36.867925 24232 consensus_peers.cc:487] T 00000000000000000000000000000000 P ceef4bfa72a4468aad391f4034246263 -> Peer eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177): Couldn't send request to peer eb210d252711436ab28b2ae64acb48a3. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.253:45177: connect: Connection refused (error 111). This is attempt 21: this message will repeat every 5th retry.
06:33:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.025204 25314 flags.cc:425] Enabled unsafe flag: --openssl_security_level_override=0
06:33:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.025470 25314 flags.cc:425] Enabled unsafe flag: --rpc_server_allow_ephemeral_ports=true
06:33:37.025 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.025547 25314 flags.cc:425] Enabled unsafe flag: --never_fsync=true
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033097 25314 flags.cc:425] Enabled experimental flag: --ipki_ca_key_size=768
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033197 25314 flags.cc:425] Enabled experimental flag: --txn_manager_enabled=true
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033234 25314 flags.cc:425] Enabled experimental flag: --ipki_server_key_size=768
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033282 25314 flags.cc:425] Enabled experimental flag: --tsk_num_rsa_bits=512
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033326 25314 flags.cc:425] Enabled experimental flag: --rpc_reuseport=true
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033389 25314 flags.cc:425] Enabled experimental flag: --enable_jwt_token_auth=true
06:33:37.033 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.033422 25314 flags.cc:425] Enabled experimental flag: --jwks_url=http://localhost:41539/jwks.json
06:33:37.042 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.042706 25314 master_runner.cc:386] Master server non-default flags:
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_poll_interval_ms=100
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --builtin_ntp_servers=127.23.161.212:44483
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ntp_initial_sync_wait_secs=10
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --time_source=builtin
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --raft_heartbeat_interval_ms=100
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_data_dirs=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --fs_wal_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_ca_key_size=768
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --master_addresses=127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --txn_manager_enabled=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --ipki_server_key_size=768
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --openssl_security_level_override=0
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --tsk_num_rsa_bits=512
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_bind_addresses=127.23.161.253:45177
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_reuseport=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --rpc_server_allow_ephemeral_ports=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_jwt_token_auth=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --jwks_url=http://localhost:41539/jwks.json
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --metrics_log_interval_ms=1000
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_format=pb
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --server_dump_info_path=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_interface=127.23.161.253
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --webserver_port=46215
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --never_fsync=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --redact=none
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_experimental_flags=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --unlock_unsafe_flags=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --enable_minidumps=false
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --log_dir=/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/logs
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logbuflevel=-1
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --logtostderr=true
06:33:37.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Master server version:
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) kudu 1.18.0-SNAPSHOT
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) revision c2d25d5f57d102d9ad9869eaf208d035a22e18aa
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build type FASTDEBUG
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) built by systest at 09 Aug 2026 05:36:50 UTC on ccycloud.quasar-ooxuuz.root.comops.site
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) build id 4575857
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) ASAN enabled
06:33:37.044 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.044067 25314 env_posix.cc:2256] Not raising this process' open files per process limit of 1048576; it is already as high as it can go
06:33:37.045 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.045791 25314 file_cache.cc:493] Constructed file cache file cache with capacity 419430
06:33:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.058653 25322 instance_detector.cc:116] could not retrieve AWS instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.058614 25323 instance_detector.cc:116] could not retrieve Azure instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:37.058 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.058701 25325 instance_detector.cc:116] could not retrieve OpenStack instance metadata: Network error: curl error: HTTP response code said error: The requested URL returned error: 404
06:33:37.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.059378 25314 server_base.cc:1034] running on GCE node
06:33:37.060 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.060240 25314 hybrid_clock.cc:584] initializing the hybrid clock with 'builtin' time source
06:33:37.061 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.061786 25314 hybrid_clock.cc:630] waiting up to --ntp_initial_sync_wait_secs=10 seconds for the clock to synchronize
06:33:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.063043 25314 hybrid_clock.cc:648] HybridClock initialized: now 1786257217062986 us; error 54 us; skew 500 ppm
06:33:37.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.063493 25314 server_base.cc:834] Flag tcmalloc_max_total_thread_cache_bytes is not working since tcmalloc is not enabled.
06:33:37.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.065796 25314 webserver.cc:460] Webserver started at http://127.23.161.253:46215/ using document root <none> and password file <none>
06:33:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.066500 25314 fs_manager.cc:359] Metadata directory not provided
06:33:37.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.066638 25314 fs_manager.cc:365] Using write-ahead log directory (fs_wal_dir) as metadata directory
06:33:37.072 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.071916 25314 fs_manager.cc:709] Time spent opening directory manager: real 0.003s user 0.003s sys 0.000s
06:33:37.075 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.074939 25331 log_block_manager.cc:3774] Time spent loading block containers with low live blocks: real 0.000s user 0.000s sys 0.000s
06:33:37.076 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.076287 25314 fs_manager.cc:725] Time spent opening block manager: real 0.003s user 0.003s sys 0.000s
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.076505 25314 fs_manager.cc:642] Opened local filesystem: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data,/tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) format_stamp: "Formatted at 2026-08-09 06:33:28 on dist-test-slave-1jjb"
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.077451 25314 fs_report.cc:383] FS layout report
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) --------------------
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) wal directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) metadata directory: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/wal
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 1 data directories: /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/data
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live blocks: 0
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes: 0
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total live bytes (after alignment): 0
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total number of LBM containers: 0 (0 full)
06:33:37.077 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for missing blocks
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Did not check for orphaned blocks
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM containers with extra space: 0 (0 repaired)
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total full LBM container extra space in bytes: 0 (0 repaired)
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total incomplete LBM containers: 0 (0 repaired)
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total LBM partial records: 0 (0 repaired)
06:33:37.078 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Total corrupted LBM metadata records in RocksDB: 0 (0 repaired)
06:33:37.106 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.106776 25314 rpc_server.cc:225] running with OpenSSL 1.0.2k-fips 26 Jan 2017
06:33:37.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.107518 25314 env_posix.cc:2256] Not raising this process' running threads per effective uid limit of 18446744073709551615; it is already as high as it can go
06:33:37.107 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.107820 25314 kserver.cc:163] Server-wide thread pool size limit: 3276
06:33:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.129841 25314 rpc_server.cc:307] RPC server started. Bound to: 127.23.161.253:45177
06:33:37.130 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.129889 25401 acceptor_pool.cc:254] collecting diagnostics on the listening RPC socket 127.23.161.253:45177 every 8 connection(s)
06:33:37.131 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.131425 25314 server_base.cc:1166] Dumped server information to /tmp/dist-test-taskbZXLUY/test-tmp/mini-kudu-cluster6876497569253959193/master-1/data/info.pb
06:33:37.137 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.137676 25402 sys_catalog.cc:263] Verifying existing consensus state
06:33:37.139 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.139765 24199 external_mini_cluster.cc:1388] Started /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu as pid 25314
06:33:37.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:33:37.140 [INFO - Time-limited test] (MiniKuduCluster.java:399) Killing master server 127.23.161.254:38015
06:33:37.140 [DEBUG - Time-limited test] (MiniKuduCluster.java:185) Request: stop_daemon {
id {
type: MASTER
index: 0
}
}
06:33:37.140 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.140798 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24212
06:33:37.141 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.140954 25402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Bootstrap starting.
06:33:37.142 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.142064 24289 generic_service.cc:196] Checking for leaks (request via RPC)
06:33:37.159 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.159024 25402 log.cc:824] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Log is configured to *not* fsync() on all Append() calls
06:33:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.171924 25402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Bootstrap replayed 1/1 log segments. Stats: ops{read=12 overwritten=0 applied=12 ignored=0} inserts{seen=8 ignored=0} mutations{seen=6 ignored=0} orphaned_commits=0. Pending: 0 replicates
06:33:37.172 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.172405 25402 tablet_bootstrap.cc:492] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: Bootstrap complete.
06:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.180784 25402 raft_consensus.cc:348] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Replica starting. Triggering 0 pending ops. Active config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.181 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.181768 25402 raft_consensus.cc:729] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Becoming Follower/Learner. State: Replica: eb210d252711436ab28b2ae64acb48a3, State: Initialized, Role: FOLLOWER
06:33:37.182 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.182415 25402 consensus_queue.cc:260] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [NON_LEADER]: Queue going to NON_LEADER mode. State: All replicated index: 0, Majority replicated index: 0, Committed index: 12, Last appended: 2.12, Last appended by leader: 12, Current term: 0, Majority size: -1, State: 0, Mode: NON_LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.183322 25406 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: RaftConsensus started. Latest consensus state: current_term: 2 committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.183 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.183694 25406 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:37.184 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.184530 25402 sys_catalog.cc:564] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: configured and running, proceeding with master startup.
06:33:37.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.194330 25418 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:37.194 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.194440 25418 catalog_manager.cc:1552] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: loading cluster ID for follower catalog manager: success
06:33:37.198 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.198675 25418 catalog_manager.cc:1574] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: acquiring CA information for follower catalog manager: success
06:33:37.201 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.201351 25418 catalog_manager.cc:1602] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3: importing token verification keys for follower catalog manager: success; most recent TSK sequence number 0
06:33:37.273 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.273238 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24212
06:33:37.301 [DEBUG - kudu-nio-1] (Connection.java:239) [peer master-127.23.161.254:38015(127.23.161.254:38015)] handling channelInactive
06:33:37.302 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state READY due to: connection closed
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.303043 25420 negotiation.cc:337] Failed RPC negotiation. Trace:
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.191696 (+ 0us) reactor.cc:625] Submitting negotiation task for client connection to 127.23.161.254:38015 (local address 127.0.0.1:36444)
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.202353 (+ 10657us) negotiation.cc:107] Waiting for socket to connect
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.202408 (+ 55us) client_negotiation.cc:174] Beginning negotiation
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.202939 (+ 531us) client_negotiation.cc:252] Sending NEGOTIATE NegotiatePB request
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.301525 (+ 98586us) negotiation.cc:327] Negotiation complete: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:33:37.303 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"client-negotiator.queue_time_us":10099,"thread_start_us":9875,"threads_started":1}
06:33:37.304 [DEBUG - Time-limited test] (MiniKuduCluster.java:195) Response:
06:33:37.304 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.304525 25170 connection.cc:535] server connection from 127.0.0.1:40762 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:33:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.305184 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: BlockingRecv error: recv error from unknown peer: Transport endpoint is not connected (error 107),
06:33:37.312 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.312255 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:37.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.450 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.23.161.254:38015(127.23.161.254:38015) for tablet Kudu Master: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.253:45177(127.23.161.253:45177)] connecting to peer
06:33:37.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.470 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.470 [DEBUG - kudu-nio-0] (Connection.java:757) [peer master-127.23.161.253:45177(127.23.161.253:45177)] Successfully connected to peer
06:33:37.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.491 [DEBUG - kudu-nio-0] (Negotiator.java:919) Authenticated connection [id: 0x9d3d64be, L:/127.0.0.1:35806 - R:/127.23.161.253:45177] using TOKEN/null
06:33:37.495 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.510 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.529 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.529371 25406 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Starting pre-election (no leader contacted us within the election timeout)
06:33:37.530 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.529726 25406 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.532 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.532218 25406 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 3 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:37.533 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.533 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.534 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.537 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.537011 25333 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:37.541 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.541509 24859 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:37.542 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.542416 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 3 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.545889 25351 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "93e8c8ec161e4104b5dc58fa5a609f99" instance_seqno: 1786257210734327) as {username='slave'} at 127.23.161.194:47827; Asking this server to re-register.
06:33:37.546 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.546636 24859 heartbeater.cc:464] Registering TS with master...
06:33:37.547 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.547164 25199 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "eb210d252711436ab28b2ae64acb48a3" candidate_term: 3 candidate_status { last_received { term: 2 index: 12 } } ignore_live_leader: false dest_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" is_pre_election: true
06:33:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.548174 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 3 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.548511 25351 ts_manager.cc:194] Registered new tserver with Master: 93e8c8ec161e4104b5dc58fa5a609f99 (127.23.161.194:45449)
06:33:37.548 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.548578 25406 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Leader pre-election lost for term 3. Reason: could not achieve majority
06:33:37.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.574227 25431 raft_consensus.cc:482] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Starting pre-election (detected failure of leader ceef4bfa72a4468aad391f4034246263)
06:33:37.574 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.574554 25431 raft_consensus.cc:504] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.576500 25431 leader_election.cc:288] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177)
06:33:37.577 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.577558 25168 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:37.579 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.579452 25168 leader_election.cc:334] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.585078 25364 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "eb210d252711436ab28b2ae64acb48a3" is_pre_election: true
06:33:37.585 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.585556 25364 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Leader pre-election vote request: Granting yes vote for candidate d573f1a12d4c4086bbf20f8e73481ad7 in term 2.
06:33:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.586123 25170 leader_election.cc:302] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 pre-election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d573f1a12d4c4086bbf20f8e73481ad7, eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263
06:33:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.586549 25431 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Leader pre-election won for term 4
06:33:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.586639 25431 raft_consensus.cc:482] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Starting leader election (detected failure of leader ceef4bfa72a4468aad391f4034246263)
06:33:37.586 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.586695 25431 raft_consensus.cc:3037] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 3 FOLLOWER]: Advancing to term 4
06:33:37.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.588789 25431 raft_consensus.cc:504] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 4 FOLLOWER]: Starting leader election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.589 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.589457 25431 leader_election.cc:288] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 election: Requested vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), eb210d252711436ab28b2ae64acb48a3 (127.23.161.253:45177)
06:33:37.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.591 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.591572 25168 leader_election.cc:334] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:37.591 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.591770 25364 tablet_service.cc:1812] Received RequestConsensusVote() RPC: tablet_id: "00000000000000000000000000000000" candidate_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" candidate_term: 4 candidate_status { last_received { term: 3 index: 13 } } ignore_live_leader: false dest_uuid: "eb210d252711436ab28b2ae64acb48a3"
06:33:37.592 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.591992 25364 raft_consensus.cc:3037] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 2 FOLLOWER]: Advancing to term 4
06:33:37.593 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.596271 25364 raft_consensus.cc:2454] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader election vote request: Granting yes vote for candidate d573f1a12d4c4086bbf20f8e73481ad7 in term 4.
06:33:37.596 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.596678 25170 leader_election.cc:302] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [CANDIDATE]: Term 4 election: Election decided. Result: candidate won. Election summary: received 3 responses out of 3 voters: 2 yes votes; 1 no votes. yes voters: d573f1a12d4c4086bbf20f8e73481ad7, eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263
06:33:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.596954 25431 raft_consensus.cc:2781] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 4 FOLLOWER]: Leader election won for term 4
06:33:37.597 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.597533 25431 raft_consensus.cc:686] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [term 4 LEADER]: Becoming Leader. State: Replica: d573f1a12d4c4086bbf20f8e73481ad7, State: Running, Role: LEADER
06:33:37.598 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.598049 25431 consensus_queue.cc:237] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [LEADER]: Queue going to LEADER mode. State: All replicated index: 0, Majority replicated index: 13, Committed index: 13, Last appended: 3.13, Last appended by leader: 13, Current term: 4, Majority size: 2, State: 0, Mode: LEADER, active raft config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:37.599 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.599646 25436 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d573f1a12d4c4086bbf20f8e73481ad7. Latest consensus state: current_term: 4 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.599897 25436 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: LEADER
06:33:37.600 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.600502 25438 catalog_manager.cc:1475] Loading table and tablet metadata into memory...
06:33:37.602 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.602825 25438 catalog_manager.cc:669] Loaded metadata for table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6]
06:33:37.603 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.603384 25438 catalog_manager.cc:669] Loaded metadata for table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74]
06:33:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.605048 25438 tablet_loader.cc:96] loaded metadata for tablet 11b35c262ebf4dd2b9e78d6abc3dea00 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.605509 25438 tablet_loader.cc:96] loaded metadata for tablet 22852a0334cf48fbb5c4242fe8272a67 (table txn_manager_fallback_rolling [id=8dc94f51511443ff887786bd502f6c74])
06:33:37.605 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.605840 25438 tablet_loader.cc:96] loaded metadata for tablet d636f709eafc42c19dfac7bd06c74d05 (table kudu_system.kudu_transactions [id=6a351c05e24745489b68bd1590153af6])
06:33:37.606 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.606112 25438 catalog_manager.cc:1484] Initializing Kudu cluster ID...
06:33:37.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.606948 25438 catalog_manager.cc:1259] Loaded cluster ID: dc044bcd6e0441d6b8e211514ae73674
06:33:37.607 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.607067 25438 catalog_manager.cc:1495] Initializing Kudu internal certificate authority...
06:33:37.608 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.608233 25438 catalog_manager.cc:1504] Loading token signing keys...
06:33:37.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.609 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.609167 25438 catalog_manager.cc:5879] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7: Loaded TSK: 0
06:33:37.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.610 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.610577 25438 catalog_manager.cc:1514] Initializing in-progress tserver states...
06:33:37.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:37.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:37.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:37.670 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2538) Learned about tablet Kudu Master for table 'Kudu Master' with partition [<start>, <end>)
06:33:37.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1488) Retrying sending RPC KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, [2ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [93ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, deferred=Deferred@406032875(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1004485492@b2a2d9e -> (continuation of Deferred@931347994 after retry RPC after error@1134413768) -> (continuation of Deferred@1193650841 after retry RPC after error@490285374) -> (continuation of Deferred@862234477 after retry RPC after error@518690126) -> (continuation of Deferred@1953600872 after retry RPC after error@542454273) -> (continuation of Deferred@916235945 after retry RPC after error@2081822161) -> (continuation of Deferred@70522350 after retry RPC after error@226032211), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/92133007@5e82bcbb -> (continuation of Deferred@931347994 after retry RPC after error@1134413768) -> (continuation of Deferred@1193650841 after retry RPC after error@490285374) -> (continuation of Deferred@862234477 after retry RPC after error@518690126) -> (continuation of Deferred@1953600872 after retry RPC after error@542454273) -> (continuation of Deferred@916235945 after retry RPC after error@2081822161) -> (continuation of Deferred@70522350 after retry RPC after error@226032211))) after lookup
06:33:37.708 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=Kudu Master, attempt=9, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, [2ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] delaying RPC due to: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [47ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [48ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [93ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [93ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [144ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [145ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: OK, [223ms] sending RPC to server master-127.23.161.252:42373, [224ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=Deferred@406032875(state=PENDING, result=null, callback=org.apache.kudu.client.KuduTransaction$$Lambda$83/1004485492@b2a2d9e -> (continuation of Deferred@931347994 after retry RPC after error@1134413768) -> (continuation of Deferred@1193650841 after retry RPC after error@490285374) -> (continuation of Deferred@862234477 after retry RPC after error@518690126) -> (continuation of Deferred@1953600872 after retry RPC after error@542454273) -> (continuation of Deferred@916235945 after retry RPC after error@2081822161) -> (continuation of Deferred@70522350 after retry RPC after error@226032211) -> (continuation of Deferred@289299911 after retry RPC@1516494971), errback=org.apache.kudu.client.KuduTransaction$$Lambda$84/92133007@5e82bcbb -> (continuation of Deferred@931347994 after retry RPC after error@1134413768) -> (continuation of Deferred@1193650841 after retry RPC after error@490285374) -> (continuation of Deferred@862234477 after retry RPC after error@518690126) -> (continuation of Deferred@1953600872 after retry RPC after error@542454273) -> (continuation of Deferred@916235945 after retry RPC after error@2081822161) -> (continuation of Deferred@70522350 after retry RPC after error@226032211) -> (continuation of Deferred@289299911 after retry RPC@1516494971)))
06:33:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.708451 25222 rpcz_store.cc:267] Call kudu.transactions.TxnManagerService.KeepTransactionAlive from 127.0.0.1:40748 (request call id 12) took 36 ms (client timeout 27 ms). Trace:
06:33:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.708681 25222 rpcz_store.cc:269] 0809 06:33:37.672336 (+ 0us) service_pool.cc:170] Inserting onto call queue
06:33:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.672410 (+ 74us) service_pool.cc:227] Handling call
06:33:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 0809 06:33:37.708380 (+ 35970us) inbound_call.cc:173] Queueing success response
06:33:37.709 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) Metrics: {"dns_us":203,"init.queue_time_us":594,"thread_start_us":1413,"threads_started":5}
06:33:37.710 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.710566 25168 consensus_peers.cc:487] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 -> Peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Couldn't send request to peer ceef4bfa72a4468aad391f4034246263. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:33:37.711 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.711262 25364 raft_consensus.cc:1261] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Refusing update from remote peer d573f1a12d4c4086bbf20f8e73481ad7: Log matching property violated. Preceding OpId in replica: term: 2 index: 12. Preceding OpId from leader: term: 4 index: 14. (index mismatch)
06:33:37.712 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.712335 25436 consensus_queue.cc:1035] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [LEADER]: Connected to new peer: Peer: permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 }, Status: LMP_MISMATCH, Last received: 0.0, Next index: 14, Last known committed idx: 12, Time since last communication: 0.000s
06:33:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.716944 25406 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: New leader d573f1a12d4c4086bbf20f8e73481ad7. Latest consensus state: current_term: 4 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.717237 25406 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.717505 25448 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.717 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.717846 25448 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.718487 25406 sys_catalog.cc:455] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: SysCatalogTable state changed. Reason: Replicated consensus-only round. Latest consensus state: current_term: 4 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.718 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.718748 25406 sys_catalog.cc:458] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [sys.catalog]: This master's current role is: FOLLOWER
06:33:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.722748 25431 sys_catalog.cc:455] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: SysCatalogTable state changed. Reason: Peer health change. Latest consensus state: current_term: 4 leader_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" committed_config { opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } } }
06:33:37.723 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.723062 25431 sys_catalog.cc:458] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 [sys.catalog]: This master's current role is: LEADER
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:37.729005 25184 catalog_manager.cc:1908] Servicing CreateTable request from {username='slave'} at 127.0.0.1:40788:
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "kudu_system.kudu_transactions"
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) schema {
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT64
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.729 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "entry_type"
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: INT8
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "identifier"
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: true
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "metadata"
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) type: STRING
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_key: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) is_nullable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) encoding: AUTO_ENCODING
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) compression: DEFAULT_COMPRESSION
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) cfile_block_size: 0
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) immutable: false
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) num_replicas: 3
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) split_rows_range_bounds {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) rows: "<redacted>""\006\001\000\000\000\000\000\000\000\000\007\001@B\017\000\000\000\000\000"
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) indirect_data: "<redacted>"""
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) partition_schema {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) range_schema {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) columns {
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) name: "txn_id"
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) }
06:33:37.730 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) table_type: TXN_STATUS_TABLE
06:33:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28: runtime error: signed integer overflow: 387927414701876 + 9222984109440078889 cannot be represented in type 'long'
06:33:37.731 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:37.731804 25184 catalog_manager.cc:6854] The number of live tablet servers is not enough to re-replicate a tablet replica of the newly created table kudu_system.kudu_transactions in case of a server failure: 4 tablet servers would be needed, 3 are available. Consider bringing up more tablet servers.
06:33:38.057 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.056237 24704 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:38.059 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.059459 25351 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "56f0ce12c25a4723a04412921d35ce43" instance_seqno: 1786257210255485) as {username='slave'} at 127.23.161.193:43467; Asking this server to re-register.
06:33:38.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.060997 24704 heartbeater.cc:464] Registering TS with master...
06:33:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.062698 25013 heartbeater.cc:347] Connected to a master server at 127.23.161.253:45177
06:33:38.063 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.063139 25351 ts_manager.cc:194] Registered new tserver with Master: 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599)
06:33:38.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.064568 25351 master_service.cc:432] Got heartbeat from unknown tserver (permanent_uuid: "371ed9133c3e433eaa40f20eb74d3876" instance_seqno: 1786257211234083) as {username='slave'} at 127.23.161.195:55827; Asking this server to re-register.
06:33:38.065 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.065438 25013 heartbeater.cc:464] Registering TS with master...
06:33:38.067 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:38.066973 25351 ts_manager.cc:194] Registered new tserver with Master: 371ed9133c3e433eaa40f20eb74d3876 (127.23.161.195:41803)
06:33:38.243 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:38.242777 25168 consensus_peers.cc:487] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 -> Peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Couldn't send request to peer ceef4bfa72a4468aad391f4034246263. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111). This is attempt 6: this message will repeat every 5th retry.
06:33:38.721 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:38.720943 25168 consensus_peers.cc:487] T 00000000000000000000000000000000 P d573f1a12d4c4086bbf20f8e73481ad7 -> Peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Couldn't send request to peer ceef4bfa72a4468aad391f4034246263. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111). This is attempt 11: this message will repeat every 5th retry.
06:33:38.804 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #0 0x7f255705a538 in kudu::operator+(kudu::MonoTime const&, kudu::MonoDelta const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28
06:33:38.856 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #1 0x9dd255 in kudu::CountDownLatch::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/countdown_latch.h:99:38
06:33:38.950 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #2 0x7f256a0cabe2 in kudu::Promise<kudu::Status>::WaitFor(kudu::MonoDelta const&) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/promise.h:57:16
06:33:38.951 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #3 0x7f256a0bb7d4 in kudu::master::Master::WaitForTxnManagerInit(kudu::MonoTime) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/master.cc:458:48
06:33:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #4 0x7f256a21016b in kudu::transactions::TxnManager::CheckInitialized(kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:283:21
06:33:38.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #5 0x7f256a210d5b in kudu::transactions::TxnManager::KeepTransactionAlive(long, std::string const&, kudu::MonoTime const&) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager.cc:226:3
06:33:38.962 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #6 0x7f256a2159cf in kudu::transactions::TxnManagerServiceImpl::KeepTransactionAlive(kudu::transactions::KeepTransactionAliveRequestPB const*, kudu::transactions::KeepTransactionAliveResponsePB*, kudu::rpc::RpcContext*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/master/txn_manager_service.cc:159:42
06:33:38.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #7 0x7f25642c6cc6 in kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const /data0/somelongdirectorytoavoidrpathissues/src/kudu/build/asan/src/kudu/master/txn_manager.service.cc:271:13
06:33:38.975 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #8 0x7f25642c69f5 in std::_Function_handler<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*), kudu::transactions::TxnManagerServiceIf::TxnManagerServiceIf(scoped_refptr<kudu::MetricEntity> const&, scoped_refptr<kudu::rpc::ResultTracker> const&)::$_9>::_M_invoke(std::_Any_data const&, google::protobuf::Message const*&&, google::protobuf::Message*&&, kudu::rpc::RpcContext*&&) ../../../include/c++/8/bits/std_function.h:297:2
06:33:38.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #9 0x7f255dc379be in std::function<void (google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*)>::operator()(google::protobuf::Message const*, google::protobuf::Message*, kudu::rpc::RpcContext*) const ../../../include/c++/8/bits/std_function.h:687:14
06:33:38.998 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #10 0x7f255dc3679e in kudu::rpc::GeneratedServiceIf::Handle(kudu::rpc::InboundCall*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_if.cc:137:3
06:33:39.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #11 0x7f255dc3cb25 in kudu::rpc::ServicePool::RunThread() /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:231:15
06:33:39.004 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #12 0x7f255dc3d54f in kudu::rpc::ServicePool::Init(int)::$_0::operator()() const /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/rpc/service_pool.cc:92:5
06:33:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #13 0x7f255dc3d378 in std::_Function_handler<void (), kudu::rpc::ServicePool::Init(int)::$_0>::_M_invoke(std::_Any_data const&) ../../../include/c++/8/bits/std_function.h:297:2
06:33:39.005 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #14 0x9edb7c in std::function<void ()>::operator()() const ../../../include/c++/8/bits/std_function.h:687:14
06:33:39.011 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #15 0x7f255711feae in kudu::Thread::SuperviseThread(void*) /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/thread.cc:694:3
06:33:39.012 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #16 0x7f255a3f96da in start_thread (/lib/x86_64-linux-gnu/libpthread.so.0+0x76da)
06:33:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) #17 0x7f255338f61e in clone (/lib/x86_64-linux-gnu/libc.so.6+0x12161e)
06:33:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751)
06:33:39.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) SUMMARY: UndefinedBehaviorSanitizer: undefined-behavior /data0/somelongdirectorytoavoidrpathissues/src/kudu/src/kudu/util/monotime.cc:332:28 in
06:33:39.034 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.034315 24702 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:39.043 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.161.252:42373(127.23.161.252:42373)] handling channelInactive
06:33:39.043 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.042976 24563 connection.cc:535] client connection to 127.23.161.252:42373 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:33:39.043 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state READY due to: connection closed
06:33:39.043 [INFO - kudu-nio-0] (AsyncKuduClient.java:2355) Invalidating location master-127.23.161.252:42373(127.23.161.252:42373) for tablet Kudu Master: connection closed
06:33:39.048 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:39.062 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.062175 25011 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:39.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.266259 25462 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:39.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.266414 25462 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:39.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.267071 25462 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.269186 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.269368 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.269490 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:39.269 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.269797 25462 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:39.304 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.305 [DEBUG - Time-limited test] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.305 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.305 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.305 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.306 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.306 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.306 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.307 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.370 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.370 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.371 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.372 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [44ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@328095569(state=PENDING, result=null, callback=(continuation of Deferred@1023971619 after retry RPC after error@35496452) -> (continuation of Deferred@1028467159 after retry RPC after error@1545127408) -> (continuation of Deferred@635487961 after retry RPC after error@1241664995) -> (continuation of Deferred@508937957 after retry RPC after error@83404698) -> (continuation of Deferred@655196114 after retry RPC after error@2142782341) -> (continuation of Deferred@1340788857 after retry RPC after error@250575465) -> (continuation of Deferred@1112565641 after retry RPC after error@2143920173), errback=(continuation of Deferred@1023971619 after retry RPC after error@35496452) -> (continuation of Deferred@1028467159 after retry RPC after error@1545127408) -> (continuation of Deferred@635487961 after retry RPC after error@1241664995) -> (continuation of Deferred@508937957 after retry RPC after error@83404698) -> (continuation of Deferred@655196114 after retry RPC after error@2142782341) -> (continuation of Deferred@1340788857 after retry RPC after error@250575465) -> (continuation of Deferred@1112565641 after retry RPC after error@2143920173)))
06:33:39.512 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] refreshing cache from master, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [44ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@328095569(state=PENDING, result=null, callback=(continuation of Deferred@1023971619 after retry RPC after error@35496452) -> (continuation of Deferred@1028467159 after retry RPC after error@1545127408) -> (continuation of Deferred@635487961 after retry RPC after error@1241664995) -> (continuation of Deferred@508937957 after retry RPC after error@83404698) -> (continuation of Deferred@655196114 after retry RPC after error@2142782341) -> (continuation of Deferred@1340788857 after retry RPC after error@250575465) -> (continuation of Deferred@1112565641 after retry RPC after error@2143920173), errback=(continuation of Deferred@1023971619 after retry RPC after error@35496452) -> (continuation of Deferred@1028467159 after retry RPC after error@1545127408) -> (continuation of Deferred@635487961 after retry RPC after error@1241664995) -> (continuation of Deferred@508937957 after retry RPC after error@83404698) -> (continuation of Deferred@655196114 after retry RPC after error@2142782341) -> (continuation of Deferred@1340788857 after retry RPC after error@250575465) -> (continuation of Deferred@1112565641 after retry RPC after error@2143920173)))
06:33:39.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.530 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.534 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.534149 24857 heartbeater.cc:503] Master 127.23.161.252:42373 was elected leader, sending a full tablet report...
06:33:39.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.650 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.650 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.651 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.652 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.693876 25462 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.694082 25462 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:39.694 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.694718 25462 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.696062 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.696281 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.696455 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:39.696 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.696735 25462 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:39.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1643028001(state=PENDING, result=null, callback=(continuation of Deferred@305877955 after retry RPC after error@1148071223) -> (continuation of Deferred@1410916703 after retry RPC after error@1091727979) -> (continuation of Deferred@159352001 after retry RPC after error@1020073804) -> (continuation of Deferred@801901459 after retry RPC after error@2099229995) -> (continuation of Deferred@145981381 after retry RPC after error@1544584936) -> (continuation of Deferred@323970666 after retry RPC after error@335101893) -> (continuation of Deferred@2565928 after retry RPC after error@1604316922) -> (continuation of Deferred@931905365 after retry RPC after error@1834400118), errback=(continuation of Deferred@305877955 after retry RPC after error@1148071223) -> (continuation of Deferred@1410916703 after retry RPC after error@1091727979) -> (continuation of Deferred@159352001 after retry RPC after error@1020073804) -> (continuation of Deferred@801901459 after retry RPC after error@2099229995) -> (continuation of Deferred@145981381 after retry RPC after error@1544584936) -> (continuation of Deferred@323970666 after retry RPC after error@335101893) -> (continuation of Deferred@2565928 after retry RPC after error@1604316922) -> (continuation of Deferred@931905365 after retry RPC after error@1834400118)))
06:33:39.772 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1643028001(state=PENDING, result=null, callback=(continuation of Deferred@305877955 after retry RPC after error@1148071223) -> (continuation of Deferred@1410916703 after retry RPC after error@1091727979) -> (continuation of Deferred@159352001 after retry RPC after error@1020073804) -> (continuation of Deferred@801901459 after retry RPC after error@2099229995) -> (continuation of Deferred@145981381 after retry RPC after error@1544584936) -> (continuation of Deferred@323970666 after retry RPC after error@335101893) -> (continuation of Deferred@2565928 after retry RPC after error@1604316922) -> (continuation of Deferred@931905365 after retry RPC after error@1834400118), errback=(continuation of Deferred@305877955 after retry RPC after error@1148071223) -> (continuation of Deferred@1410916703 after retry RPC after error@1091727979) -> (continuation of Deferred@159352001 after retry RPC after error@1020073804) -> (continuation of Deferred@801901459 after retry RPC after error@2099229995) -> (continuation of Deferred@145981381 after retry RPC after error@1544584936) -> (continuation of Deferred@323970666 after retry RPC after error@335101893) -> (continuation of Deferred@2565928 after retry RPC after error@1604316922) -> (continuation of Deferred@931905365 after retry RPC after error@1834400118)))
06:33:39.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.828 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.852 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.910 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.910 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.910 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.950 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.951 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.951 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.952 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.952 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.953 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.958 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:39.958177 24883 txn_status_manager.cc:1390] automatically aborted stale txn (ID 0) past 1.139s from last keepalive heartbeat (effective timeout is 1.000s)
06:33:39.959 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:39.959785 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:33:39.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:39.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [164ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1013671188(state=PENDING, result=null, callback=(continuation of Deferred@603765010 after retry RPC after error@1093439922) -> (continuation of Deferred@714411867 after retry RPC after error@1295092952) -> (continuation of Deferred@197363781 after retry RPC after error@784804238) -> (continuation of Deferred@749339866 after retry RPC after error@1841044187) -> (continuation of Deferred@2049586825 after retry RPC after error@1832315675) -> (continuation of Deferred@298233523 after retry RPC after error@471930002) -> (continuation of Deferred@932509665 after retry RPC after error@1839246395), errback=(continuation of Deferred@603765010 after retry RPC after error@1093439922) -> (continuation of Deferred@714411867 after retry RPC after error@1295092952) -> (continuation of Deferred@197363781 after retry RPC after error@784804238) -> (continuation of Deferred@749339866 after retry RPC after error@1841044187) -> (continuation of Deferred@2049586825 after retry RPC after error@1832315675) -> (continuation of Deferred@298233523 after retry RPC after error@471930002) -> (continuation of Deferred@932509665 after retry RPC after error@1839246395)))
06:33:39.972 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [164ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [165ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1013671188(state=PENDING, result=null, callback=(continuation of Deferred@603765010 after retry RPC after error@1093439922) -> (continuation of Deferred@714411867 after retry RPC after error@1295092952) -> (continuation of Deferred@197363781 after retry RPC after error@784804238) -> (continuation of Deferred@749339866 after retry RPC after error@1841044187) -> (continuation of Deferred@2049586825 after retry RPC after error@1832315675) -> (continuation of Deferred@298233523 after retry RPC after error@471930002) -> (continuation of Deferred@932509665 after retry RPC after error@1839246395), errback=(continuation of Deferred@603765010 after retry RPC after error@1093439922) -> (continuation of Deferred@714411867 after retry RPC after error@1295092952) -> (continuation of Deferred@197363781 after retry RPC after error@784804238) -> (continuation of Deferred@749339866 after retry RPC after error@1841044187) -> (continuation of Deferred@2049586825 after retry RPC after error@1832315675) -> (continuation of Deferred@298233523 after retry RPC after error@471930002) -> (continuation of Deferred@932509665 after retry RPC after error@1839246395)))
06:33:39.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:39.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:39.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:39.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:39.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.000505 25462 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:40.000 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.000691 25462 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:40.001 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.001351 25462 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:40.002543 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:40.002713 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:40.002 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.002838 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:40.003 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.003063 25462 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:40.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.249 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@981166459(state=PENDING, result=null, callback=(continuation of Deferred@802950669 after retry RPC after error@1366754866) -> (continuation of Deferred@1261008033 after retry RPC after error@720960526) -> (continuation of Deferred@1704665914 after retry RPC after error@204799789) -> (continuation of Deferred@213393014 after retry RPC after error@1640864611) -> (continuation of Deferred@1576269459 after retry RPC after error@1981130647) -> (continuation of Deferred@974277773 after retry RPC after error@931782191) -> (continuation of Deferred@1963778339 after retry RPC after error@788453642), errback=(continuation of Deferred@802950669 after retry RPC after error@1366754866) -> (continuation of Deferred@1261008033 after retry RPC after error@720960526) -> (continuation of Deferred@1704665914 after retry RPC after error@204799789) -> (continuation of Deferred@213393014 after retry RPC after error@1640864611) -> (continuation of Deferred@1576269459 after retry RPC after error@1981130647) -> (continuation of Deferred@974277773 after retry RPC after error@931782191) -> (continuation of Deferred@1963778339 after retry RPC after error@788453642)))
06:33:40.250 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:40.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.290 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.292 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.310 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.312 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.431 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1683494221(state=PENDING, result=null, callback=(continuation of Deferred@847056666 after retry RPC after error@1792966904) -> (continuation of Deferred@1392246505 after retry RPC after error@1991080838) -> (continuation of Deferred@745717654 after retry RPC after error@842343328) -> (continuation of Deferred@1068797881 after retry RPC after error@576238210) -> (continuation of Deferred@460269392 after retry RPC after error@6041534) -> (continuation of Deferred@453943627 after retry RPC after error@2091244554) -> (continuation of Deferred@512897385 after retry RPC after error@2136136922), errback=(continuation of Deferred@847056666 after retry RPC after error@1792966904) -> (continuation of Deferred@1392246505 after retry RPC after error@1991080838) -> (continuation of Deferred@745717654 after retry RPC after error@842343328) -> (continuation of Deferred@1068797881 after retry RPC after error@576238210) -> (continuation of Deferred@460269392 after retry RPC after error@6041534) -> (continuation of Deferred@453943627 after retry RPC after error@2091244554) -> (continuation of Deferred@512897385 after retry RPC after error@2136136922)))
06:33:40.431 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1683494221(state=PENDING, result=null, callback=(continuation of Deferred@847056666 after retry RPC after error@1792966904) -> (continuation of Deferred@1392246505 after retry RPC after error@1991080838) -> (continuation of Deferred@745717654 after retry RPC after error@842343328) -> (continuation of Deferred@1068797881 after retry RPC after error@576238210) -> (continuation of Deferred@460269392 after retry RPC after error@6041534) -> (continuation of Deferred@453943627 after retry RPC after error@2091244554) -> (continuation of Deferred@512897385 after retry RPC after error@2136136922), errback=(continuation of Deferred@847056666 after retry RPC after error@1792966904) -> (continuation of Deferred@1392246505 after retry RPC after error@1991080838) -> (continuation of Deferred@745717654 after retry RPC after error@842343328) -> (continuation of Deferred@1068797881 after retry RPC after error@576238210) -> (continuation of Deferred@460269392 after retry RPC after error@6041534) -> (continuation of Deferred@453943627 after retry RPC after error@2091244554) -> (continuation of Deferred@512897385 after retry RPC after error@2136136922)))
06:33:40.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.610 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1383932193(state=PENDING, result=null, callback=(continuation of Deferred@1664350196 after retry RPC after error@1279389567) -> (continuation of Deferred@1198001042 after retry RPC after error@2091339383) -> (continuation of Deferred@5741120 after retry RPC after error@1127135010) -> (continuation of Deferred@1151144014 after retry RPC after error@34548316) -> (continuation of Deferred@1043736007 after retry RPC after error@144118404) -> (continuation of Deferred@614755512 after retry RPC after error@1594032058), errback=(continuation of Deferred@1664350196 after retry RPC after error@1279389567) -> (continuation of Deferred@1198001042 after retry RPC after error@2091339383) -> (continuation of Deferred@5741120 after retry RPC after error@1127135010) -> (continuation of Deferred@1151144014 after retry RPC after error@34548316) -> (continuation of Deferred@1043736007 after retry RPC after error@144118404) -> (continuation of Deferred@614755512 after retry RPC after error@1594032058)))
06:33:40.612 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1383932193(state=PENDING, result=null, callback=(continuation of Deferred@1664350196 after retry RPC after error@1279389567) -> (continuation of Deferred@1198001042 after retry RPC after error@2091339383) -> (continuation of Deferred@5741120 after retry RPC after error@1127135010) -> (continuation of Deferred@1151144014 after retry RPC after error@34548316) -> (continuation of Deferred@1043736007 after retry RPC after error@144118404) -> (continuation of Deferred@614755512 after retry RPC after error@1594032058), errback=(continuation of Deferred@1664350196 after retry RPC after error@1279389567) -> (continuation of Deferred@1198001042 after retry RPC after error@2091339383) -> (continuation of Deferred@5741120 after retry RPC after error@1127135010) -> (continuation of Deferred@1151144014 after retry RPC after error@34548316) -> (continuation of Deferred@1043736007 after retry RPC after error@144118404) -> (continuation of Deferred@614755512 after retry RPC after error@1594032058)))
06:33:40.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.690 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1620099614(state=PENDING, result=null, callback=(continuation of Deferred@750318264 after retry RPC after error@2017994828) -> (continuation of Deferred@961118773 after retry RPC after error@1596929840) -> (continuation of Deferred@879309663 after retry RPC after error@232417238) -> (continuation of Deferred@2029458546 after retry RPC after error@1590115051) -> (continuation of Deferred@600381863 after retry RPC after error@2113089247) -> (continuation of Deferred@1030218135 after retry RPC after error@1519424690), errback=(continuation of Deferred@750318264 after retry RPC after error@2017994828) -> (continuation of Deferred@961118773 after retry RPC after error@1596929840) -> (continuation of Deferred@879309663 after retry RPC after error@232417238) -> (continuation of Deferred@2029458546 after retry RPC after error@1590115051) -> (continuation of Deferred@600381863 after retry RPC after error@2113089247) -> (continuation of Deferred@1030218135 after retry RPC after error@1519424690)))
06:33:40.812 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1620099614(state=PENDING, result=null, callback=(continuation of Deferred@750318264 after retry RPC after error@2017994828) -> (continuation of Deferred@961118773 after retry RPC after error@1596929840) -> (continuation of Deferred@879309663 after retry RPC after error@232417238) -> (continuation of Deferred@2029458546 after retry RPC after error@1590115051) -> (continuation of Deferred@600381863 after retry RPC after error@2113089247) -> (continuation of Deferred@1030218135 after retry RPC after error@1519424690), errback=(continuation of Deferred@750318264 after retry RPC after error@2017994828) -> (continuation of Deferred@961118773 after retry RPC after error@1596929840) -> (continuation of Deferred@879309663 after retry RPC after error@232417238) -> (continuation of Deferred@2029458546 after retry RPC after error@1590115051) -> (continuation of Deferred@600381863 after retry RPC after error@2113089247) -> (continuation of Deferred@1030218135 after retry RPC after error@1519424690)))
06:33:40.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:40.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:40.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:40.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:40.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:40.970 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.970580 25489 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.970744 25489 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:40.971 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.971505 25489 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:40.973 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:40.973681 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:40.973969 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.974134 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:40.974 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:40.974396 25489 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:41.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.070 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.070 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.072 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1160606974(state=PENDING, result=null, callback=(continuation of Deferred@1346931572 after retry RPC after error@1897078149) -> (continuation of Deferred@795233461 after retry RPC after error@21120803) -> (continuation of Deferred@365260628 after retry RPC after error@2085857116) -> (continuation of Deferred@1632456583 after retry RPC after error@1923442642) -> (continuation of Deferred@1277430045 after retry RPC after error@766262914) -> (continuation of Deferred@647396807 after retry RPC after error@2144605488) -> (continuation of Deferred@324306742 after retry RPC after error@1598092272), errback=(continuation of Deferred@1346931572 after retry RPC after error@1897078149) -> (continuation of Deferred@795233461 after retry RPC after error@21120803) -> (continuation of Deferred@365260628 after retry RPC after error@2085857116) -> (continuation of Deferred@1632456583 after retry RPC after error@1923442642) -> (continuation of Deferred@1277430045 after retry RPC after error@766262914) -> (continuation of Deferred@647396807 after retry RPC after error@2144605488) -> (continuation of Deferred@324306742 after retry RPC after error@1598092272)))
06:33:41.072 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1160606974(state=PENDING, result=null, callback=(continuation of Deferred@1346931572 after retry RPC after error@1897078149) -> (continuation of Deferred@795233461 after retry RPC after error@21120803) -> (continuation of Deferred@365260628 after retry RPC after error@2085857116) -> (continuation of Deferred@1632456583 after retry RPC after error@1923442642) -> (continuation of Deferred@1277430045 after retry RPC after error@766262914) -> (continuation of Deferred@647396807 after retry RPC after error@2144605488) -> (continuation of Deferred@324306742 after retry RPC after error@1598092272), errback=(continuation of Deferred@1346931572 after retry RPC after error@1897078149) -> (continuation of Deferred@795233461 after retry RPC after error@21120803) -> (continuation of Deferred@365260628 after retry RPC after error@2085857116) -> (continuation of Deferred@1632456583 after retry RPC after error@1923442642) -> (continuation of Deferred@1277430045 after retry RPC after error@766262914) -> (continuation of Deferred@647396807 after retry RPC after error@2144605488) -> (continuation of Deferred@324306742 after retry RPC after error@1598092272)))
06:33:41.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.150 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.150 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.170 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.251 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@38323618(state=PENDING, result=null, callback=(continuation of Deferred@425051220 after retry RPC after error@2019888110) -> (continuation of Deferred@859828618 after retry RPC after error@1998477210) -> (continuation of Deferred@1088313929 after retry RPC after error@271848391) -> (continuation of Deferred@1556812778 after retry RPC after error@1500261742) -> (continuation of Deferred@1808934936 after retry RPC after error@1531487374) -> (continuation of Deferred@1603844174 after retry RPC after error@1730673038) -> (continuation of Deferred@491753370 after retry RPC after error@54338665), errback=(continuation of Deferred@425051220 after retry RPC after error@2019888110) -> (continuation of Deferred@859828618 after retry RPC after error@1998477210) -> (continuation of Deferred@1088313929 after retry RPC after error@271848391) -> (continuation of Deferred@1556812778 after retry RPC after error@1500261742) -> (continuation of Deferred@1808934936 after retry RPC after error@1531487374) -> (continuation of Deferred@1603844174 after retry RPC after error@1730673038) -> (continuation of Deferred@491753370 after retry RPC after error@54338665)))
06:33:41.252 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@38323618(state=PENDING, result=null, callback=(continuation of Deferred@425051220 after retry RPC after error@2019888110) -> (continuation of Deferred@859828618 after retry RPC after error@1998477210) -> (continuation of Deferred@1088313929 after retry RPC after error@271848391) -> (continuation of Deferred@1556812778 after retry RPC after error@1500261742) -> (continuation of Deferred@1808934936 after retry RPC after error@1531487374) -> (continuation of Deferred@1603844174 after retry RPC after error@1730673038) -> (continuation of Deferred@491753370 after retry RPC after error@54338665), errback=(continuation of Deferred@425051220 after retry RPC after error@2019888110) -> (continuation of Deferred@859828618 after retry RPC after error@1998477210) -> (continuation of Deferred@1088313929 after retry RPC after error@271848391) -> (continuation of Deferred@1556812778 after retry RPC after error@1500261742) -> (continuation of Deferred@1808934936 after retry RPC after error@1531487374) -> (continuation of Deferred@1603844174 after retry RPC after error@1730673038) -> (continuation of Deferred@491753370 after retry RPC after error@54338665)))
06:33:41.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.510 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.512 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1600051535(state=PENDING, result=null, callback=(continuation of Deferred@953484231 after retry RPC after error@91879996) -> (continuation of Deferred@1242444852 after retry RPC after error@1672677531) -> (continuation of Deferred@1944646673 after retry RPC after error@956979274) -> (continuation of Deferred@43939019 after retry RPC after error@2039628755) -> (continuation of Deferred@1299085452 after retry RPC after error@642621795) -> (continuation of Deferred@1459925325 after retry RPC after error@1013181605) -> (continuation of Deferred@41372093 after retry RPC after error@1232778793), errback=(continuation of Deferred@953484231 after retry RPC after error@91879996) -> (continuation of Deferred@1242444852 after retry RPC after error@1672677531) -> (continuation of Deferred@1944646673 after retry RPC after error@956979274) -> (continuation of Deferred@43939019 after retry RPC after error@2039628755) -> (continuation of Deferred@1299085452 after retry RPC after error@642621795) -> (continuation of Deferred@1459925325 after retry RPC after error@1013181605) -> (continuation of Deferred@41372093 after retry RPC after error@1232778793)))
06:33:41.512 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1600051535(state=PENDING, result=null, callback=(continuation of Deferred@953484231 after retry RPC after error@91879996) -> (continuation of Deferred@1242444852 after retry RPC after error@1672677531) -> (continuation of Deferred@1944646673 after retry RPC after error@956979274) -> (continuation of Deferred@43939019 after retry RPC after error@2039628755) -> (continuation of Deferred@1299085452 after retry RPC after error@642621795) -> (continuation of Deferred@1459925325 after retry RPC after error@1013181605) -> (continuation of Deferred@41372093 after retry RPC after error@1232778793), errback=(continuation of Deferred@953484231 after retry RPC after error@91879996) -> (continuation of Deferred@1242444852 after retry RPC after error@1672677531) -> (continuation of Deferred@1944646673 after retry RPC after error@956979274) -> (continuation of Deferred@43939019 after retry RPC after error@2039628755) -> (continuation of Deferred@1299085452 after retry RPC after error@642621795) -> (continuation of Deferred@1459925325 after retry RPC after error@1013181605) -> (continuation of Deferred@41372093 after retry RPC after error@1232778793)))
06:33:41.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.690 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.690 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.690 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2051352590(state=PENDING, result=null, callback=(continuation of Deferred@731732289 after retry RPC after error@1858244810) -> (continuation of Deferred@2022441389 after retry RPC after error@727340010) -> (continuation of Deferred@1791770769 after retry RPC after error@275958210) -> (continuation of Deferred@149631752 after retry RPC after error@1901585660) -> (continuation of Deferred@2040003215 after retry RPC after error@2142044326) -> (continuation of Deferred@772742952 after retry RPC after error@1613852177), errback=(continuation of Deferred@731732289 after retry RPC after error@1858244810) -> (continuation of Deferred@2022441389 after retry RPC after error@727340010) -> (continuation of Deferred@1791770769 after retry RPC after error@275958210) -> (continuation of Deferred@149631752 after retry RPC after error@1901585660) -> (continuation of Deferred@2040003215 after retry RPC after error@2142044326) -> (continuation of Deferred@772742952 after retry RPC after error@1613852177)))
06:33:41.692 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2051352590(state=PENDING, result=null, callback=(continuation of Deferred@731732289 after retry RPC after error@1858244810) -> (continuation of Deferred@2022441389 after retry RPC after error@727340010) -> (continuation of Deferred@1791770769 after retry RPC after error@275958210) -> (continuation of Deferred@149631752 after retry RPC after error@1901585660) -> (continuation of Deferred@2040003215 after retry RPC after error@2142044326) -> (continuation of Deferred@772742952 after retry RPC after error@1613852177), errback=(continuation of Deferred@731732289 after retry RPC after error@1858244810) -> (continuation of Deferred@2022441389 after retry RPC after error@727340010) -> (continuation of Deferred@1791770769 after retry RPC after error@275958210) -> (continuation of Deferred@149631752 after retry RPC after error@1901585660) -> (continuation of Deferred@2040003215 after retry RPC after error@2142044326) -> (continuation of Deferred@772742952 after retry RPC after error@1613852177)))
06:33:41.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@393259974(state=PENDING, result=null, callback=(continuation of Deferred@1371480161 after retry RPC after error@2023639997) -> (continuation of Deferred@1047801517 after retry RPC after error@742956094) -> (continuation of Deferred@81530366 after retry RPC after error@236285307) -> (continuation of Deferred@294333266 after retry RPC after error@2034957361) -> (continuation of Deferred@488484123 after retry RPC after error@952526198) -> (continuation of Deferred@1893668999 after retry RPC after error@1591396825), errback=(continuation of Deferred@1371480161 after retry RPC after error@2023639997) -> (continuation of Deferred@1047801517 after retry RPC after error@742956094) -> (continuation of Deferred@81530366 after retry RPC after error@236285307) -> (continuation of Deferred@294333266 after retry RPC after error@2034957361) -> (continuation of Deferred@488484123 after retry RPC after error@952526198) -> (continuation of Deferred@1893668999 after retry RPC after error@1591396825)))
06:33:41.851 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@393259974(state=PENDING, result=null, callback=(continuation of Deferred@1371480161 after retry RPC after error@2023639997) -> (continuation of Deferred@1047801517 after retry RPC after error@742956094) -> (continuation of Deferred@81530366 after retry RPC after error@236285307) -> (continuation of Deferred@294333266 after retry RPC after error@2034957361) -> (continuation of Deferred@488484123 after retry RPC after error@952526198) -> (continuation of Deferred@1893668999 after retry RPC after error@1591396825), errback=(continuation of Deferred@1371480161 after retry RPC after error@2023639997) -> (continuation of Deferred@1047801517 after retry RPC after error@742956094) -> (continuation of Deferred@81530366 after retry RPC after error@236285307) -> (continuation of Deferred@294333266 after retry RPC after error@2034957361) -> (continuation of Deferred@488484123 after retry RPC after error@952526198) -> (continuation of Deferred@1893668999 after retry RPC after error@1591396825)))
06:33:41.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:41.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:41.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:41.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:41.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:41.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.050 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.051 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.051 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@143000879(state=PENDING, result=null, callback=(continuation of Deferred@1097601948 after retry RPC after error@1058594319) -> (continuation of Deferred@1360321624 after retry RPC after error@1770198056) -> (continuation of Deferred@1213336183 after retry RPC after error@834887124) -> (continuation of Deferred@1178771880 after retry RPC after error@932302620) -> (continuation of Deferred@1875409451 after retry RPC after error@1146005499) -> (continuation of Deferred@399642939 after retry RPC after error@780326386) -> (continuation of Deferred@45611819 after retry RPC after error@863495673), errback=(continuation of Deferred@1097601948 after retry RPC after error@1058594319) -> (continuation of Deferred@1360321624 after retry RPC after error@1770198056) -> (continuation of Deferred@1213336183 after retry RPC after error@834887124) -> (continuation of Deferred@1178771880 after retry RPC after error@932302620) -> (continuation of Deferred@1875409451 after retry RPC after error@1146005499) -> (continuation of Deferred@399642939 after retry RPC after error@780326386) -> (continuation of Deferred@45611819 after retry RPC after error@863495673)))
06:33:42.052 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@143000879(state=PENDING, result=null, callback=(continuation of Deferred@1097601948 after retry RPC after error@1058594319) -> (continuation of Deferred@1360321624 after retry RPC after error@1770198056) -> (continuation of Deferred@1213336183 after retry RPC after error@834887124) -> (continuation of Deferred@1178771880 after retry RPC after error@932302620) -> (continuation of Deferred@1875409451 after retry RPC after error@1146005499) -> (continuation of Deferred@399642939 after retry RPC after error@780326386) -> (continuation of Deferred@45611819 after retry RPC after error@863495673), errback=(continuation of Deferred@1097601948 after retry RPC after error@1058594319) -> (continuation of Deferred@1360321624 after retry RPC after error@1770198056) -> (continuation of Deferred@1213336183 after retry RPC after error@834887124) -> (continuation of Deferred@1178771880 after retry RPC after error@932302620) -> (continuation of Deferred@1875409451 after retry RPC after error@1146005499) -> (continuation of Deferred@399642939 after retry RPC after error@780326386) -> (continuation of Deferred@45611819 after retry RPC after error@863495673)))
06:33:42.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.251778 25502 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:42.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.251998 25502 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:42.252 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.252750 25502 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:42.254920 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:42.255098 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.255203 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:42.255 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.255419 25502 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:42.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.311 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.311 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1814857756(state=PENDING, result=null, callback=(continuation of Deferred@633323613 after retry RPC after error@754966144) -> (continuation of Deferred@2004258893 after retry RPC after error@2131925719) -> (continuation of Deferred@619970624 after retry RPC after error@1982714240) -> (continuation of Deferred@890946544 after retry RPC after error@1580540838) -> (continuation of Deferred@1514655264 after retry RPC after error@1115600185) -> (continuation of Deferred@563669957 after retry RPC after error@1455936579) -> (continuation of Deferred@848635509 after retry RPC after error@91589219), errback=(continuation of Deferred@633323613 after retry RPC after error@754966144) -> (continuation of Deferred@2004258893 after retry RPC after error@2131925719) -> (continuation of Deferred@619970624 after retry RPC after error@1982714240) -> (continuation of Deferred@890946544 after retry RPC after error@1580540838) -> (continuation of Deferred@1514655264 after retry RPC after error@1115600185) -> (continuation of Deferred@563669957 after retry RPC after error@1455936579) -> (continuation of Deferred@848635509 after retry RPC after error@91589219)))
06:33:42.312 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1814857756(state=PENDING, result=null, callback=(continuation of Deferred@633323613 after retry RPC after error@754966144) -> (continuation of Deferred@2004258893 after retry RPC after error@2131925719) -> (continuation of Deferred@619970624 after retry RPC after error@1982714240) -> (continuation of Deferred@890946544 after retry RPC after error@1580540838) -> (continuation of Deferred@1514655264 after retry RPC after error@1115600185) -> (continuation of Deferred@563669957 after retry RPC after error@1455936579) -> (continuation of Deferred@848635509 after retry RPC after error@91589219), errback=(continuation of Deferred@633323613 after retry RPC after error@754966144) -> (continuation of Deferred@2004258893 after retry RPC after error@2131925719) -> (continuation of Deferred@619970624 after retry RPC after error@1982714240) -> (continuation of Deferred@890946544 after retry RPC after error@1580540838) -> (continuation of Deferred@1514655264 after retry RPC after error@1115600185) -> (continuation of Deferred@563669957 after retry RPC after error@1455936579) -> (continuation of Deferred@848635509 after retry RPC after error@91589219)))
06:33:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.313928 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:33:42.314 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:42.314111 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:42.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.510 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@313202035(state=PENDING, result=null, callback=(continuation of Deferred@1311954066 after retry RPC after error@2015660814) -> (continuation of Deferred@242892442 after retry RPC after error@1144038059) -> (continuation of Deferred@486435067 after retry RPC after error@1079966865) -> (continuation of Deferred@529503470 after retry RPC after error@1592477774) -> (continuation of Deferred@826100440 after retry RPC after error@103842330) -> (continuation of Deferred@1702584022 after retry RPC after error@1641468687) -> (continuation of Deferred@979775079 after retry RPC after error@944003939), errback=(continuation of Deferred@1311954066 after retry RPC after error@2015660814) -> (continuation of Deferred@242892442 after retry RPC after error@1144038059) -> (continuation of Deferred@486435067 after retry RPC after error@1079966865) -> (continuation of Deferred@529503470 after retry RPC after error@1592477774) -> (continuation of Deferred@826100440 after retry RPC after error@103842330) -> (continuation of Deferred@1702584022 after retry RPC after error@1641468687) -> (continuation of Deferred@979775079 after retry RPC after error@944003939)))
06:33:42.511 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@313202035(state=PENDING, result=null, callback=(continuation of Deferred@1311954066 after retry RPC after error@2015660814) -> (continuation of Deferred@242892442 after retry RPC after error@1144038059) -> (continuation of Deferred@486435067 after retry RPC after error@1079966865) -> (continuation of Deferred@529503470 after retry RPC after error@1592477774) -> (continuation of Deferred@826100440 after retry RPC after error@103842330) -> (continuation of Deferred@1702584022 after retry RPC after error@1641468687) -> (continuation of Deferred@979775079 after retry RPC after error@944003939), errback=(continuation of Deferred@1311954066 after retry RPC after error@2015660814) -> (continuation of Deferred@242892442 after retry RPC after error@1144038059) -> (continuation of Deferred@486435067 after retry RPC after error@1079966865) -> (continuation of Deferred@529503470 after retry RPC after error@1592477774) -> (continuation of Deferred@826100440 after retry RPC after error@103842330) -> (continuation of Deferred@1702584022 after retry RPC after error@1641468687) -> (continuation of Deferred@979775079 after retry RPC after error@944003939)))
06:33:42.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.750 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1855105504(state=PENDING, result=null, callback=(continuation of Deferred@542560470 after retry RPC after error@660246490) -> (continuation of Deferred@1623044349 after retry RPC after error@463194275) -> (continuation of Deferred@232614758 after retry RPC after error@2122977624) -> (continuation of Deferred@1933409324 after retry RPC after error@916876344) -> (continuation of Deferred@1355196369 after retry RPC after error@2129165376) -> (continuation of Deferred@1714205613 after retry RPC after error@1668207868) -> (continuation of Deferred@116596591 after retry RPC after error@1002731917), errback=(continuation of Deferred@542560470 after retry RPC after error@660246490) -> (continuation of Deferred@1623044349 after retry RPC after error@463194275) -> (continuation of Deferred@232614758 after retry RPC after error@2122977624) -> (continuation of Deferred@1933409324 after retry RPC after error@916876344) -> (continuation of Deferred@1355196369 after retry RPC after error@2129165376) -> (continuation of Deferred@1714205613 after retry RPC after error@1668207868) -> (continuation of Deferred@116596591 after retry RPC after error@1002731917)))
06:33:42.752 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1855105504(state=PENDING, result=null, callback=(continuation of Deferred@542560470 after retry RPC after error@660246490) -> (continuation of Deferred@1623044349 after retry RPC after error@463194275) -> (continuation of Deferred@232614758 after retry RPC after error@2122977624) -> (continuation of Deferred@1933409324 after retry RPC after error@916876344) -> (continuation of Deferred@1355196369 after retry RPC after error@2129165376) -> (continuation of Deferred@1714205613 after retry RPC after error@1668207868) -> (continuation of Deferred@116596591 after retry RPC after error@1002731917), errback=(continuation of Deferred@542560470 after retry RPC after error@660246490) -> (continuation of Deferred@1623044349 after retry RPC after error@463194275) -> (continuation of Deferred@232614758 after retry RPC after error@2122977624) -> (continuation of Deferred@1933409324 after retry RPC after error@916876344) -> (continuation of Deferred@1355196369 after retry RPC after error@2129165376) -> (continuation of Deferred@1714205613 after retry RPC after error@1668207868) -> (continuation of Deferred@116596591 after retry RPC after error@1002731917)))
06:33:42.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:42.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:42.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:42.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.908 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:42.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:42.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.029 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2032867093(state=PENDING, result=null, callback=(continuation of Deferred@1547743431 after retry RPC after error@151714432) -> (continuation of Deferred@1421473102 after retry RPC after error@1972093811) -> (continuation of Deferred@747930509 after retry RPC after error@1922347621) -> (continuation of Deferred@1771842300 after retry RPC after error@1109912807) -> (continuation of Deferred@1252493253 after retry RPC after error@494343022) -> (continuation of Deferred@355729910 after retry RPC after error@1745459043) -> (continuation of Deferred@518433800 after retry RPC after error@992944345), errback=(continuation of Deferred@1547743431 after retry RPC after error@151714432) -> (continuation of Deferred@1421473102 after retry RPC after error@1972093811) -> (continuation of Deferred@747930509 after retry RPC after error@1922347621) -> (continuation of Deferred@1771842300 after retry RPC after error@1109912807) -> (continuation of Deferred@1252493253 after retry RPC after error@494343022) -> (continuation of Deferred@355729910 after retry RPC after error@1745459043) -> (continuation of Deferred@518433800 after retry RPC after error@992944345)))
06:33:43.029 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:43.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.072 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.231 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [24ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1502791808(state=PENDING, result=null, callback=(continuation of Deferred@1722575922 after retry RPC after error@1310472121) -> (continuation of Deferred@1137668547 after retry RPC after error@206993394) -> (continuation of Deferred@161372086 after retry RPC after error@638589123) -> (continuation of Deferred@518595755 after retry RPC after error@557787491) -> (continuation of Deferred@1516759317 after retry RPC after error@2090284186) -> (continuation of Deferred@722242527 after retry RPC after error@1939917804), errback=(continuation of Deferred@1722575922 after retry RPC after error@1310472121) -> (continuation of Deferred@1137668547 after retry RPC after error@206993394) -> (continuation of Deferred@161372086 after retry RPC after error@638589123) -> (continuation of Deferred@518595755 after retry RPC after error@557787491) -> (continuation of Deferred@1516759317 after retry RPC after error@2090284186) -> (continuation of Deferred@722242527 after retry RPC after error@1939917804)))
06:33:43.232 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [24ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [24ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1502791808(state=PENDING, result=null, callback=(continuation of Deferred@1722575922 after retry RPC after error@1310472121) -> (continuation of Deferred@1137668547 after retry RPC after error@206993394) -> (continuation of Deferred@161372086 after retry RPC after error@638589123) -> (continuation of Deferred@518595755 after retry RPC after error@557787491) -> (continuation of Deferred@1516759317 after retry RPC after error@2090284186) -> (continuation of Deferred@722242527 after retry RPC after error@1939917804), errback=(continuation of Deferred@1722575922 after retry RPC after error@1310472121) -> (continuation of Deferred@1137668547 after retry RPC after error@206993394) -> (continuation of Deferred@161372086 after retry RPC after error@638589123) -> (continuation of Deferred@518595755 after retry RPC after error@557787491) -> (continuation of Deferred@1516759317 after retry RPC after error@2090284186) -> (continuation of Deferred@722242527 after retry RPC after error@1939917804)))
06:33:43.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.310 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.311 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.471 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1678338996(state=PENDING, result=null, callback=(continuation of Deferred@109993892 after retry RPC after error@1304460363) -> (continuation of Deferred@1694785276 after retry RPC after error@1258278179) -> (continuation of Deferred@1048107175 after retry RPC after error@1880414471) -> (continuation of Deferred@933272135 after retry RPC after error@50519877) -> (continuation of Deferred@2068284002 after retry RPC after error@760062188) -> (continuation of Deferred@504133024 after retry RPC after error@199326163) -> (continuation of Deferred@1282588255 after retry RPC after error@209202685), errback=(continuation of Deferred@109993892 after retry RPC after error@1304460363) -> (continuation of Deferred@1694785276 after retry RPC after error@1258278179) -> (continuation of Deferred@1048107175 after retry RPC after error@1880414471) -> (continuation of Deferred@933272135 after retry RPC after error@50519877) -> (continuation of Deferred@2068284002 after retry RPC after error@760062188) -> (continuation of Deferred@504133024 after retry RPC after error@199326163) -> (continuation of Deferred@1282588255 after retry RPC after error@209202685)))
06:33:43.472 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1678338996(state=PENDING, result=null, callback=(continuation of Deferred@109993892 after retry RPC after error@1304460363) -> (continuation of Deferred@1694785276 after retry RPC after error@1258278179) -> (continuation of Deferred@1048107175 after retry RPC after error@1880414471) -> (continuation of Deferred@933272135 after retry RPC after error@50519877) -> (continuation of Deferred@2068284002 after retry RPC after error@760062188) -> (continuation of Deferred@504133024 after retry RPC after error@199326163) -> (continuation of Deferred@1282588255 after retry RPC after error@209202685), errback=(continuation of Deferred@109993892 after retry RPC after error@1304460363) -> (continuation of Deferred@1694785276 after retry RPC after error@1258278179) -> (continuation of Deferred@1048107175 after retry RPC after error@1880414471) -> (continuation of Deferred@933272135 after retry RPC after error@50519877) -> (continuation of Deferred@2068284002 after retry RPC after error@760062188) -> (continuation of Deferred@504133024 after retry RPC after error@199326163) -> (continuation of Deferred@1282588255 after retry RPC after error@209202685)))
06:33:43.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.731 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1547705196(state=PENDING, result=null, callback=(continuation of Deferred@1714304729 after retry RPC after error@960139034) -> (continuation of Deferred@1288671503 after retry RPC after error@277379201) -> (continuation of Deferred@126291127 after retry RPC after error@1781010690) -> (continuation of Deferred@1516216993 after retry RPC after error@163611931) -> (continuation of Deferred@907924000 after retry RPC after error@471299552) -> (continuation of Deferred@997885545 after retry RPC after error@916399168) -> (continuation of Deferred@1858825313 after retry RPC after error@1230285869), errback=(continuation of Deferred@1714304729 after retry RPC after error@960139034) -> (continuation of Deferred@1288671503 after retry RPC after error@277379201) -> (continuation of Deferred@126291127 after retry RPC after error@1781010690) -> (continuation of Deferred@1516216993 after retry RPC after error@163611931) -> (continuation of Deferred@907924000 after retry RPC after error@471299552) -> (continuation of Deferred@997885545 after retry RPC after error@916399168) -> (continuation of Deferred@1858825313 after retry RPC after error@1230285869)))
06:33:43.732 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1547705196(state=PENDING, result=null, callback=(continuation of Deferred@1714304729 after retry RPC after error@960139034) -> (continuation of Deferred@1288671503 after retry RPC after error@277379201) -> (continuation of Deferred@126291127 after retry RPC after error@1781010690) -> (continuation of Deferred@1516216993 after retry RPC after error@163611931) -> (continuation of Deferred@907924000 after retry RPC after error@471299552) -> (continuation of Deferred@997885545 after retry RPC after error@916399168) -> (continuation of Deferred@1858825313 after retry RPC after error@1230285869), errback=(continuation of Deferred@1714304729 after retry RPC after error@960139034) -> (continuation of Deferred@1288671503 after retry RPC after error@277379201) -> (continuation of Deferred@126291127 after retry RPC after error@1781010690) -> (continuation of Deferred@1516216993 after retry RPC after error@163611931) -> (continuation of Deferred@907924000 after retry RPC after error@471299552) -> (continuation of Deferred@997885545 after retry RPC after error@916399168) -> (continuation of Deferred@1858825313 after retry RPC after error@1230285869)))
06:33:43.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.890 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.931 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1893539238(state=PENDING, result=null, callback=(continuation of Deferred@2139670543 after retry RPC after error@1690518644) -> (continuation of Deferred@582419626 after retry RPC after error@983502199) -> (continuation of Deferred@65954459 after retry RPC after error@2112906870) -> (continuation of Deferred@1735966518 after retry RPC after error@634315291) -> (continuation of Deferred@1430566531 after retry RPC after error@737786251) -> (continuation of Deferred@156902093 after retry RPC after error@1532417991) -> (continuation of Deferred@727089792 after retry RPC after error@1292190592), errback=(continuation of Deferred@2139670543 after retry RPC after error@1690518644) -> (continuation of Deferred@582419626 after retry RPC after error@983502199) -> (continuation of Deferred@65954459 after retry RPC after error@2112906870) -> (continuation of Deferred@1735966518 after retry RPC after error@634315291) -> (continuation of Deferred@1430566531 after retry RPC after error@737786251) -> (continuation of Deferred@156902093 after retry RPC after error@1532417991) -> (continuation of Deferred@727089792 after retry RPC after error@1292190592)))
06:33:43.932 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1893539238(state=PENDING, result=null, callback=(continuation of Deferred@2139670543 after retry RPC after error@1690518644) -> (continuation of Deferred@582419626 after retry RPC after error@983502199) -> (continuation of Deferred@65954459 after retry RPC after error@2112906870) -> (continuation of Deferred@1735966518 after retry RPC after error@634315291) -> (continuation of Deferred@1430566531 after retry RPC after error@737786251) -> (continuation of Deferred@156902093 after retry RPC after error@1532417991) -> (continuation of Deferred@727089792 after retry RPC after error@1292190592), errback=(continuation of Deferred@2139670543 after retry RPC after error@1690518644) -> (continuation of Deferred@582419626 after retry RPC after error@983502199) -> (continuation of Deferred@65954459 after retry RPC after error@2112906870) -> (continuation of Deferred@1735966518 after retry RPC after error@634315291) -> (continuation of Deferred@1430566531 after retry RPC after error@737786251) -> (continuation of Deferred@156902093 after retry RPC after error@1532417991) -> (continuation of Deferred@727089792 after retry RPC after error@1292190592)))
06:33:43.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:43.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:43.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:43.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:43.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:43.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.131 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1042300294(state=PENDING, result=null, callback=(continuation of Deferred@672904751 after retry RPC after error@638259335) -> (continuation of Deferred@2118501466 after retry RPC after error@1094064665) -> (continuation of Deferred@879326942 after retry RPC after error@1251526652) -> (continuation of Deferred@439332321 after retry RPC after error@1798871663) -> (continuation of Deferred@337174096 after retry RPC after error@414407688) -> (continuation of Deferred@2140453466 after retry RPC after error@1347468325) -> (continuation of Deferred@40370721 after retry RPC after error@12148088), errback=(continuation of Deferred@672904751 after retry RPC after error@638259335) -> (continuation of Deferred@2118501466 after retry RPC after error@1094064665) -> (continuation of Deferred@879326942 after retry RPC after error@1251526652) -> (continuation of Deferred@439332321 after retry RPC after error@1798871663) -> (continuation of Deferred@337174096 after retry RPC after error@414407688) -> (continuation of Deferred@2140453466 after retry RPC after error@1347468325) -> (continuation of Deferred@40370721 after retry RPC after error@12148088)))
06:33:44.131 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1042300294(state=PENDING, result=null, callback=(continuation of Deferred@672904751 after retry RPC after error@638259335) -> (continuation of Deferred@2118501466 after retry RPC after error@1094064665) -> (continuation of Deferred@879326942 after retry RPC after error@1251526652) -> (continuation of Deferred@439332321 after retry RPC after error@1798871663) -> (continuation of Deferred@337174096 after retry RPC after error@414407688) -> (continuation of Deferred@2140453466 after retry RPC after error@1347468325) -> (continuation of Deferred@40370721 after retry RPC after error@12148088), errback=(continuation of Deferred@672904751 after retry RPC after error@638259335) -> (continuation of Deferred@2118501466 after retry RPC after error@1094064665) -> (continuation of Deferred@879326942 after retry RPC after error@1251526652) -> (continuation of Deferred@439332321 after retry RPC after error@1798871663) -> (continuation of Deferred@337174096 after retry RPC after error@414407688) -> (continuation of Deferred@2140453466 after retry RPC after error@1347468325) -> (continuation of Deferred@40370721 after retry RPC after error@12148088)))
06:33:44.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.170 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.170 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.330 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.330 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@961481408(state=PENDING, result=null, callback=(continuation of Deferred@908595187 after retry RPC after error@1550067373) -> (continuation of Deferred@1729617361 after retry RPC after error@651154169) -> (continuation of Deferred@1357780548 after retry RPC after error@759301050) -> (continuation of Deferred@1961015958 after retry RPC after error@753067753) -> (continuation of Deferred@211720455 after retry RPC after error@830211325) -> (continuation of Deferred@1353399886 after retry RPC after error@1808097052) -> (continuation of Deferred@334210498 after retry RPC after error@1113657518), errback=(continuation of Deferred@908595187 after retry RPC after error@1550067373) -> (continuation of Deferred@1729617361 after retry RPC after error@651154169) -> (continuation of Deferred@1357780548 after retry RPC after error@759301050) -> (continuation of Deferred@1961015958 after retry RPC after error@753067753) -> (continuation of Deferred@211720455 after retry RPC after error@830211325) -> (continuation of Deferred@1353399886 after retry RPC after error@1808097052) -> (continuation of Deferred@334210498 after retry RPC after error@1113657518)))
06:33:44.371 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@961481408(state=PENDING, result=null, callback=(continuation of Deferred@908595187 after retry RPC after error@1550067373) -> (continuation of Deferred@1729617361 after retry RPC after error@651154169) -> (continuation of Deferred@1357780548 after retry RPC after error@759301050) -> (continuation of Deferred@1961015958 after retry RPC after error@753067753) -> (continuation of Deferred@211720455 after retry RPC after error@830211325) -> (continuation of Deferred@1353399886 after retry RPC after error@1808097052) -> (continuation of Deferred@334210498 after retry RPC after error@1113657518), errback=(continuation of Deferred@908595187 after retry RPC after error@1550067373) -> (continuation of Deferred@1729617361 after retry RPC after error@651154169) -> (continuation of Deferred@1357780548 after retry RPC after error@759301050) -> (continuation of Deferred@1961015958 after retry RPC after error@753067753) -> (continuation of Deferred@211720455 after retry RPC after error@830211325) -> (continuation of Deferred@1353399886 after retry RPC after error@1808097052) -> (continuation of Deferred@334210498 after retry RPC after error@1113657518)))
06:33:44.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:44.419572 25520 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:44.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:44.419747 25520 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:44.420 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:44.420367 25520 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:44.421 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.421231 25333 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111) [suppressed 10 similar messages]
06:33:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.422295 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.422483 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:44.422 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:44.422643 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:44.423 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:44.422904 25520 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:44.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@490293497(state=PENDING, result=null, callback=(continuation of Deferred@1099384877 after retry RPC after error@1665569511) -> (continuation of Deferred@20782635 after retry RPC after error@1444208757) -> (continuation of Deferred@766053562 after retry RPC after error@2058318402) -> (continuation of Deferred@327870330 after retry RPC after error@1651292726) -> (continuation of Deferred@130397725 after retry RPC after error@108657093) -> (continuation of Deferred@1116503879 after retry RPC after error@1169604650) -> (continuation of Deferred@1869214687 after retry RPC after error@639080486), errback=(continuation of Deferred@1099384877 after retry RPC after error@1665569511) -> (continuation of Deferred@20782635 after retry RPC after error@1444208757) -> (continuation of Deferred@766053562 after retry RPC after error@2058318402) -> (continuation of Deferred@327870330 after retry RPC after error@1651292726) -> (continuation of Deferred@130397725 after retry RPC after error@108657093) -> (continuation of Deferred@1116503879 after retry RPC after error@1169604650) -> (continuation of Deferred@1869214687 after retry RPC after error@639080486)))
06:33:44.611 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@490293497(state=PENDING, result=null, callback=(continuation of Deferred@1099384877 after retry RPC after error@1665569511) -> (continuation of Deferred@20782635 after retry RPC after error@1444208757) -> (continuation of Deferred@766053562 after retry RPC after error@2058318402) -> (continuation of Deferred@327870330 after retry RPC after error@1651292726) -> (continuation of Deferred@130397725 after retry RPC after error@108657093) -> (continuation of Deferred@1116503879 after retry RPC after error@1169604650) -> (continuation of Deferred@1869214687 after retry RPC after error@639080486), errback=(continuation of Deferred@1099384877 after retry RPC after error@1665569511) -> (continuation of Deferred@20782635 after retry RPC after error@1444208757) -> (continuation of Deferred@766053562 after retry RPC after error@2058318402) -> (continuation of Deferred@327870330 after retry RPC after error@1651292726) -> (continuation of Deferred@130397725 after retry RPC after error@108657093) -> (continuation of Deferred@1116503879 after retry RPC after error@1169604650) -> (continuation of Deferred@1869214687 after retry RPC after error@639080486)))
06:33:44.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1623045539(state=PENDING, result=null, callback=(continuation of Deferred@1034734156 after retry RPC after error@797450759) -> (continuation of Deferred@700734505 after retry RPC after error@618263583) -> (continuation of Deferred@2147267682 after retry RPC after error@294271360) -> (continuation of Deferred@438573708 after retry RPC after error@1899392219) -> (continuation of Deferred@1811126587 after retry RPC after error@729152508) -> (continuation of Deferred@297759844 after retry RPC after error@1395495484) -> (continuation of Deferred@610333475 after retry RPC after error@542004808) -> (continuation of Deferred@1253579377 after retry RPC after error@1975242088), errback=(continuation of Deferred@1034734156 after retry RPC after error@797450759) -> (continuation of Deferred@700734505 after retry RPC after error@618263583) -> (continuation of Deferred@2147267682 after retry RPC after error@294271360) -> (continuation of Deferred@438573708 after retry RPC after error@1899392219) -> (continuation of Deferred@1811126587 after retry RPC after error@729152508) -> (continuation of Deferred@297759844 after retry RPC after error@1395495484) -> (continuation of Deferred@610333475 after retry RPC after error@542004808) -> (continuation of Deferred@1253579377 after retry RPC after error@1975242088)))
06:33:44.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1623045539(state=PENDING, result=null, callback=(continuation of Deferred@1034734156 after retry RPC after error@797450759) -> (continuation of Deferred@700734505 after retry RPC after error@618263583) -> (continuation of Deferred@2147267682 after retry RPC after error@294271360) -> (continuation of Deferred@438573708 after retry RPC after error@1899392219) -> (continuation of Deferred@1811126587 after retry RPC after error@729152508) -> (continuation of Deferred@297759844 after retry RPC after error@1395495484) -> (continuation of Deferred@610333475 after retry RPC after error@542004808) -> (continuation of Deferred@1253579377 after retry RPC after error@1975242088), errback=(continuation of Deferred@1034734156 after retry RPC after error@797450759) -> (continuation of Deferred@700734505 after retry RPC after error@618263583) -> (continuation of Deferred@2147267682 after retry RPC after error@294271360) -> (continuation of Deferred@438573708 after retry RPC after error@1899392219) -> (continuation of Deferred@1811126587 after retry RPC after error@729152508) -> (continuation of Deferred@297759844 after retry RPC after error@1395495484) -> (continuation of Deferred@610333475 after retry RPC after error@542004808) -> (continuation of Deferred@1253579377 after retry RPC after error@1975242088)))
06:33:44.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:44.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:44.989 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.988866 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:44.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:44.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:44.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:44.992 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.991971 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:44.993 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:44.993559 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:33:45.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.110 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.110 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1739301164(state=PENDING, result=null, callback=(continuation of Deferred@1852689999 after retry RPC after error@511812868) -> (continuation of Deferred@816641046 after retry RPC after error@171213294) -> (continuation of Deferred@178993489 after retry RPC after error@428938373) -> (continuation of Deferred@1313492965 after retry RPC after error@1616874547) -> (continuation of Deferred@862802136 after retry RPC after error@575233804) -> (continuation of Deferred@1046481351 after retry RPC after error@583148588) -> (continuation of Deferred@2070956860 after retry RPC after error@329898593), errback=(continuation of Deferred@1852689999 after retry RPC after error@511812868) -> (continuation of Deferred@816641046 after retry RPC after error@171213294) -> (continuation of Deferred@178993489 after retry RPC after error@428938373) -> (continuation of Deferred@1313492965 after retry RPC after error@1616874547) -> (continuation of Deferred@862802136 after retry RPC after error@575233804) -> (continuation of Deferred@1046481351 after retry RPC after error@583148588) -> (continuation of Deferred@2070956860 after retry RPC after error@329898593)))
06:33:45.112 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1739301164(state=PENDING, result=null, callback=(continuation of Deferred@1852689999 after retry RPC after error@511812868) -> (continuation of Deferred@816641046 after retry RPC after error@171213294) -> (continuation of Deferred@178993489 after retry RPC after error@428938373) -> (continuation of Deferred@1313492965 after retry RPC after error@1616874547) -> (continuation of Deferred@862802136 after retry RPC after error@575233804) -> (continuation of Deferred@1046481351 after retry RPC after error@583148588) -> (continuation of Deferred@2070956860 after retry RPC after error@329898593), errback=(continuation of Deferred@1852689999 after retry RPC after error@511812868) -> (continuation of Deferred@816641046 after retry RPC after error@171213294) -> (continuation of Deferred@178993489 after retry RPC after error@428938373) -> (continuation of Deferred@1313492965 after retry RPC after error@1616874547) -> (continuation of Deferred@862802136 after retry RPC after error@575233804) -> (continuation of Deferred@1046481351 after retry RPC after error@583148588) -> (continuation of Deferred@2070956860 after retry RPC after error@329898593)))
06:33:45.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.250 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.250 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1843648427(state=PENDING, result=null, callback=(continuation of Deferred@1920077920 after retry RPC after error@952150150) -> (continuation of Deferred@1289654268 after retry RPC after error@1064523587) -> (continuation of Deferred@1575002589 after retry RPC after error@1624876671) -> (continuation of Deferred@1489003767 after retry RPC after error@2100099555) -> (continuation of Deferred@622477778 after retry RPC after error@140374352) -> (continuation of Deferred@1398922096 after retry RPC after error@1226363678), errback=(continuation of Deferred@1920077920 after retry RPC after error@952150150) -> (continuation of Deferred@1289654268 after retry RPC after error@1064523587) -> (continuation of Deferred@1575002589 after retry RPC after error@1624876671) -> (continuation of Deferred@1489003767 after retry RPC after error@2100099555) -> (continuation of Deferred@622477778 after retry RPC after error@140374352) -> (continuation of Deferred@1398922096 after retry RPC after error@1226363678)))
06:33:45.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1843648427(state=PENDING, result=null, callback=(continuation of Deferred@1920077920 after retry RPC after error@952150150) -> (continuation of Deferred@1289654268 after retry RPC after error@1064523587) -> (continuation of Deferred@1575002589 after retry RPC after error@1624876671) -> (continuation of Deferred@1489003767 after retry RPC after error@2100099555) -> (continuation of Deferred@622477778 after retry RPC after error@140374352) -> (continuation of Deferred@1398922096 after retry RPC after error@1226363678), errback=(continuation of Deferred@1920077920 after retry RPC after error@952150150) -> (continuation of Deferred@1289654268 after retry RPC after error@1064523587) -> (continuation of Deferred@1575002589 after retry RPC after error@1624876671) -> (continuation of Deferred@1489003767 after retry RPC after error@2100099555) -> (continuation of Deferred@622477778 after retry RPC after error@140374352) -> (continuation of Deferred@1398922096 after retry RPC after error@1226363678)))
06:33:45.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.388 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.389 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.490 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.510 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1587371672(state=PENDING, result=null, callback=(continuation of Deferred@1707372218 after retry RPC after error@697474240) -> (continuation of Deferred@2098831948 after retry RPC after error@1229216489) -> (continuation of Deferred@255024325 after retry RPC after error@855287167) -> (continuation of Deferred@37841582 after retry RPC after error@1773796125) -> (continuation of Deferred@2088322971 after retry RPC after error@458216502) -> (continuation of Deferred@315808235 after retry RPC after error@882227647) -> (continuation of Deferred@48657705 after retry RPC after error@1735257895), errback=(continuation of Deferred@1707372218 after retry RPC after error@697474240) -> (continuation of Deferred@2098831948 after retry RPC after error@1229216489) -> (continuation of Deferred@255024325 after retry RPC after error@855287167) -> (continuation of Deferred@37841582 after retry RPC after error@1773796125) -> (continuation of Deferred@2088322971 after retry RPC after error@458216502) -> (continuation of Deferred@315808235 after retry RPC after error@882227647) -> (continuation of Deferred@48657705 after retry RPC after error@1735257895)))
06:33:45.511 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1587371672(state=PENDING, result=null, callback=(continuation of Deferred@1707372218 after retry RPC after error@697474240) -> (continuation of Deferred@2098831948 after retry RPC after error@1229216489) -> (continuation of Deferred@255024325 after retry RPC after error@855287167) -> (continuation of Deferred@37841582 after retry RPC after error@1773796125) -> (continuation of Deferred@2088322971 after retry RPC after error@458216502) -> (continuation of Deferred@315808235 after retry RPC after error@882227647) -> (continuation of Deferred@48657705 after retry RPC after error@1735257895), errback=(continuation of Deferred@1707372218 after retry RPC after error@697474240) -> (continuation of Deferred@2098831948 after retry RPC after error@1229216489) -> (continuation of Deferred@255024325 after retry RPC after error@855287167) -> (continuation of Deferred@37841582 after retry RPC after error@1773796125) -> (continuation of Deferred@2088322971 after retry RPC after error@458216502) -> (continuation of Deferred@315808235 after retry RPC after error@882227647) -> (continuation of Deferred@48657705 after retry RPC after error@1735257895)))
06:33:45.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:45.651151 25533 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:45.651 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:45.651312 25533 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:45.652 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:45.651904 25533 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:45.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:45.654057 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:45.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:45.654253 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:45.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:45.654371 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:45.654 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:45.654608 25533 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:45.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.789 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1011617092(state=PENDING, result=null, callback=(continuation of Deferred@181829910 after retry RPC after error@335076510) -> (continuation of Deferred@122019867 after retry RPC after error@1539308442) -> (continuation of Deferred@1700732874 after retry RPC after error@806757331) -> (continuation of Deferred@419125490 after retry RPC after error@1010565980) -> (continuation of Deferred@553708126 after retry RPC after error@545869078) -> (continuation of Deferred@1660403005 after retry RPC after error@1041069276) -> (continuation of Deferred@418982745 after retry RPC after error@1920087213) -> (continuation of Deferred@766734731 after retry RPC after error@2026724213), errback=(continuation of Deferred@181829910 after retry RPC after error@335076510) -> (continuation of Deferred@122019867 after retry RPC after error@1539308442) -> (continuation of Deferred@1700732874 after retry RPC after error@806757331) -> (continuation of Deferred@419125490 after retry RPC after error@1010565980) -> (continuation of Deferred@553708126 after retry RPC after error@545869078) -> (continuation of Deferred@1660403005 after retry RPC after error@1041069276) -> (continuation of Deferred@418982745 after retry RPC after error@1920087213) -> (continuation of Deferred@766734731 after retry RPC after error@2026724213)))
06:33:45.790 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:45.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.908 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.908 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.909 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:45.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1613303047(state=PENDING, result=null, callback=(continuation of Deferred@1166643078 after retry RPC after error@659207655) -> (continuation of Deferred@547468355 after retry RPC after error@138986597) -> (continuation of Deferred@28614485 after retry RPC after error@1931732745) -> (continuation of Deferred@1577932418 after retry RPC after error@279497418) -> (continuation of Deferred@902017469 after retry RPC after error@1496581027) -> (continuation of Deferred@1832511297 after retry RPC after error@999394998) -> (continuation of Deferred@334994976 after retry RPC after error@1789374182), errback=(continuation of Deferred@1166643078 after retry RPC after error@659207655) -> (continuation of Deferred@547468355 after retry RPC after error@138986597) -> (continuation of Deferred@28614485 after retry RPC after error@1931732745) -> (continuation of Deferred@1577932418 after retry RPC after error@279497418) -> (continuation of Deferred@902017469 after retry RPC after error@1496581027) -> (continuation of Deferred@1832511297 after retry RPC after error@999394998) -> (continuation of Deferred@334994976 after retry RPC after error@1789374182)))
06:33:45.971 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1613303047(state=PENDING, result=null, callback=(continuation of Deferred@1166643078 after retry RPC after error@659207655) -> (continuation of Deferred@547468355 after retry RPC after error@138986597) -> (continuation of Deferred@28614485 after retry RPC after error@1931732745) -> (continuation of Deferred@1577932418 after retry RPC after error@279497418) -> (continuation of Deferred@902017469 after retry RPC after error@1496581027) -> (continuation of Deferred@1832511297 after retry RPC after error@999394998) -> (continuation of Deferred@334994976 after retry RPC after error@1789374182), errback=(continuation of Deferred@1166643078 after retry RPC after error@659207655) -> (continuation of Deferred@547468355 after retry RPC after error@138986597) -> (continuation of Deferred@28614485 after retry RPC after error@1931732745) -> (continuation of Deferred@1577932418 after retry RPC after error@279497418) -> (continuation of Deferred@902017469 after retry RPC after error@1496581027) -> (continuation of Deferred@1832511297 after retry RPC after error@999394998) -> (continuation of Deferred@334994976 after retry RPC after error@1789374182)))
06:33:45.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:45.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:45.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:45.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:45.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.150 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.248 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@681111369(state=PENDING, result=null, callback=(continuation of Deferred@1059032910 after retry RPC after error@1305428206) -> (continuation of Deferred@1841727623 after retry RPC after error@46907601) -> (continuation of Deferred@1106070073 after retry RPC after error@1989323482) -> (continuation of Deferred@793351743 after retry RPC after error@1935152719) -> (continuation of Deferred@1481586335 after retry RPC after error@996705078) -> (continuation of Deferred@1492320778 after retry RPC after error@1797954869) -> (continuation of Deferred@1257288961 after retry RPC after error@897511002), errback=(continuation of Deferred@1059032910 after retry RPC after error@1305428206) -> (continuation of Deferred@1841727623 after retry RPC after error@46907601) -> (continuation of Deferred@1106070073 after retry RPC after error@1989323482) -> (continuation of Deferred@793351743 after retry RPC after error@1935152719) -> (continuation of Deferred@1481586335 after retry RPC after error@996705078) -> (continuation of Deferred@1492320778 after retry RPC after error@1797954869) -> (continuation of Deferred@1257288961 after retry RPC after error@897511002)))
06:33:46.249 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:46.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.508 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@983706544(state=PENDING, result=null, callback=(continuation of Deferred@1687327093 after retry RPC after error@159664069) -> (continuation of Deferred@1789681301 after retry RPC after error@1106773979) -> (continuation of Deferred@831602600 after retry RPC after error@436272176) -> (continuation of Deferred@307249427 after retry RPC after error@328682077) -> (continuation of Deferred@734219256 after retry RPC after error@918537164) -> (continuation of Deferred@653739217 after retry RPC after error@253437936) -> (continuation of Deferred@1043738976 after retry RPC after error@1985946825) -> (continuation of Deferred@1779578655 after retry RPC after error@1021356298), errback=(continuation of Deferred@1687327093 after retry RPC after error@159664069) -> (continuation of Deferred@1789681301 after retry RPC after error@1106773979) -> (continuation of Deferred@831602600 after retry RPC after error@436272176) -> (continuation of Deferred@307249427 after retry RPC after error@328682077) -> (continuation of Deferred@734219256 after retry RPC after error@918537164) -> (continuation of Deferred@653739217 after retry RPC after error@253437936) -> (continuation of Deferred@1043738976 after retry RPC after error@1985946825) -> (continuation of Deferred@1779578655 after retry RPC after error@1021356298)))
06:33:46.511 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@983706544(state=PENDING, result=null, callback=(continuation of Deferred@1687327093 after retry RPC after error@159664069) -> (continuation of Deferred@1789681301 after retry RPC after error@1106773979) -> (continuation of Deferred@831602600 after retry RPC after error@436272176) -> (continuation of Deferred@307249427 after retry RPC after error@328682077) -> (continuation of Deferred@734219256 after retry RPC after error@918537164) -> (continuation of Deferred@653739217 after retry RPC after error@253437936) -> (continuation of Deferred@1043738976 after retry RPC after error@1985946825) -> (continuation of Deferred@1779578655 after retry RPC after error@1021356298), errback=(continuation of Deferred@1687327093 after retry RPC after error@159664069) -> (continuation of Deferred@1789681301 after retry RPC after error@1106773979) -> (continuation of Deferred@831602600 after retry RPC after error@436272176) -> (continuation of Deferred@307249427 after retry RPC after error@328682077) -> (continuation of Deferred@734219256 after retry RPC after error@918537164) -> (continuation of Deferred@653739217 after retry RPC after error@253437936) -> (continuation of Deferred@1043738976 after retry RPC after error@1985946825) -> (continuation of Deferred@1779578655 after retry RPC after error@1021356298)))
06:33:46.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.669 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.750 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1799117070(state=PENDING, result=null, callback=(continuation of Deferred@1448655407 after retry RPC after error@201474931) -> (continuation of Deferred@186839404 after retry RPC after error@48180527) -> (continuation of Deferred@289760921 after retry RPC after error@256026137) -> (continuation of Deferred@334995212 after retry RPC after error@1757052424) -> (continuation of Deferred@1391574056 after retry RPC after error@1218034177) -> (continuation of Deferred@1731810273 after retry RPC after error@1548233520) -> (continuation of Deferred@50842733 after retry RPC after error@105592766) -> (continuation of Deferred@725666509 after retry RPC after error@910976318), errback=(continuation of Deferred@1448655407 after retry RPC after error@201474931) -> (continuation of Deferred@186839404 after retry RPC after error@48180527) -> (continuation of Deferred@289760921 after retry RPC after error@256026137) -> (continuation of Deferred@334995212 after retry RPC after error@1757052424) -> (continuation of Deferred@1391574056 after retry RPC after error@1218034177) -> (continuation of Deferred@1731810273 after retry RPC after error@1548233520) -> (continuation of Deferred@50842733 after retry RPC after error@105592766) -> (continuation of Deferred@725666509 after retry RPC after error@910976318)))
06:33:46.751 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1799117070(state=PENDING, result=null, callback=(continuation of Deferred@1448655407 after retry RPC after error@201474931) -> (continuation of Deferred@186839404 after retry RPC after error@48180527) -> (continuation of Deferred@289760921 after retry RPC after error@256026137) -> (continuation of Deferred@334995212 after retry RPC after error@1757052424) -> (continuation of Deferred@1391574056 after retry RPC after error@1218034177) -> (continuation of Deferred@1731810273 after retry RPC after error@1548233520) -> (continuation of Deferred@50842733 after retry RPC after error@105592766) -> (continuation of Deferred@725666509 after retry RPC after error@910976318), errback=(continuation of Deferred@1448655407 after retry RPC after error@201474931) -> (continuation of Deferred@186839404 after retry RPC after error@48180527) -> (continuation of Deferred@289760921 after retry RPC after error@256026137) -> (continuation of Deferred@334995212 after retry RPC after error@1757052424) -> (continuation of Deferred@1391574056 after retry RPC after error@1218034177) -> (continuation of Deferred@1731810273 after retry RPC after error@1548233520) -> (continuation of Deferred@50842733 after retry RPC after error@105592766) -> (continuation of Deferred@725666509 after retry RPC after error@910976318)))
06:33:46.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.829 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:46.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:46.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:46.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:46.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:46.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1886107491(state=PENDING, result=null, callback=(continuation of Deferred@1340980788 after retry RPC after error@1738263015) -> (continuation of Deferred@958305574 after retry RPC after error@342093257) -> (continuation of Deferred@920517999 after retry RPC after error@263249570) -> (continuation of Deferred@1120591319 after retry RPC after error@1229749730) -> (continuation of Deferred@279172897 after retry RPC after error@435117657) -> (continuation of Deferred@232725718 after retry RPC after error@668390370) -> (continuation of Deferred@700580582 after retry RPC after error@1253874705), errback=(continuation of Deferred@1340980788 after retry RPC after error@1738263015) -> (continuation of Deferred@958305574 after retry RPC after error@342093257) -> (continuation of Deferred@920517999 after retry RPC after error@263249570) -> (continuation of Deferred@1120591319 after retry RPC after error@1229749730) -> (continuation of Deferred@279172897 after retry RPC after error@435117657) -> (continuation of Deferred@232725718 after retry RPC after error@668390370) -> (continuation of Deferred@700580582 after retry RPC after error@1253874705)))
06:33:46.991 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1886107491(state=PENDING, result=null, callback=(continuation of Deferred@1340980788 after retry RPC after error@1738263015) -> (continuation of Deferred@958305574 after retry RPC after error@342093257) -> (continuation of Deferred@920517999 after retry RPC after error@263249570) -> (continuation of Deferred@1120591319 after retry RPC after error@1229749730) -> (continuation of Deferred@279172897 after retry RPC after error@435117657) -> (continuation of Deferred@232725718 after retry RPC after error@668390370) -> (continuation of Deferred@700580582 after retry RPC after error@1253874705), errback=(continuation of Deferred@1340980788 after retry RPC after error@1738263015) -> (continuation of Deferred@958305574 after retry RPC after error@342093257) -> (continuation of Deferred@920517999 after retry RPC after error@263249570) -> (continuation of Deferred@1120591319 after retry RPC after error@1229749730) -> (continuation of Deferred@279172897 after retry RPC after error@435117657) -> (continuation of Deferred@232725718 after retry RPC after error@668390370) -> (continuation of Deferred@700580582 after retry RPC after error@1253874705)))
06:33:47.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.068 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.109 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.210 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@804058344(state=PENDING, result=null, callback=(continuation of Deferred@1349786282 after retry RPC after error@1338488748) -> (continuation of Deferred@128171736 after retry RPC after error@1327518064) -> (continuation of Deferred@1072804724 after retry RPC after error@1299782129) -> (continuation of Deferred@1471556600 after retry RPC after error@441473967) -> (continuation of Deferred@680010114 after retry RPC after error@758846451) -> (continuation of Deferred@1261700167 after retry RPC after error@1394713517) -> (continuation of Deferred@978062458 after retry RPC after error@1335698507), errback=(continuation of Deferred@1349786282 after retry RPC after error@1338488748) -> (continuation of Deferred@128171736 after retry RPC after error@1327518064) -> (continuation of Deferred@1072804724 after retry RPC after error@1299782129) -> (continuation of Deferred@1471556600 after retry RPC after error@441473967) -> (continuation of Deferred@680010114 after retry RPC after error@758846451) -> (continuation of Deferred@1261700167 after retry RPC after error@1394713517) -> (continuation of Deferred@978062458 after retry RPC after error@1335698507)))
06:33:47.211 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@804058344(state=PENDING, result=null, callback=(continuation of Deferred@1349786282 after retry RPC after error@1338488748) -> (continuation of Deferred@128171736 after retry RPC after error@1327518064) -> (continuation of Deferred@1072804724 after retry RPC after error@1299782129) -> (continuation of Deferred@1471556600 after retry RPC after error@441473967) -> (continuation of Deferred@680010114 after retry RPC after error@758846451) -> (continuation of Deferred@1261700167 after retry RPC after error@1394713517) -> (continuation of Deferred@978062458 after retry RPC after error@1335698507), errback=(continuation of Deferred@1349786282 after retry RPC after error@1338488748) -> (continuation of Deferred@128171736 after retry RPC after error@1327518064) -> (continuation of Deferred@1072804724 after retry RPC after error@1299782129) -> (continuation of Deferred@1471556600 after retry RPC after error@441473967) -> (continuation of Deferred@680010114 after retry RPC after error@758846451) -> (continuation of Deferred@1261700167 after retry RPC after error@1394713517) -> (continuation of Deferred@978062458 after retry RPC after error@1335698507)))
06:33:47.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.270 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.271 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.271 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.271 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.272 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:47.315267 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:33:47.315 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:47.315456 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:47.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.350 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.450 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.450 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [79ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1095538066(state=PENDING, result=null, callback=(continuation of Deferred@1243862345 after retry RPC after error@198880940) -> (continuation of Deferred@1261886513 after retry RPC after error@1861431488) -> (continuation of Deferred@976103882 after retry RPC after error@1683967981) -> (continuation of Deferred@427151363 after retry RPC after error@1861719773) -> (continuation of Deferred@635212464 after retry RPC after error@1004700097) -> (continuation of Deferred@211856246 after retry RPC after error@1588827457) -> (continuation of Deferred@532399413 after retry RPC after error@821601162), errback=(continuation of Deferred@1243862345 after retry RPC after error@198880940) -> (continuation of Deferred@1261886513 after retry RPC after error@1861431488) -> (continuation of Deferred@976103882 after retry RPC after error@1683967981) -> (continuation of Deferred@427151363 after retry RPC after error@1861719773) -> (continuation of Deferred@635212464 after retry RPC after error@1004700097) -> (continuation of Deferred@211856246 after retry RPC after error@1588827457) -> (continuation of Deferred@532399413 after retry RPC after error@821601162)))
06:33:47.451 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [79ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1095538066(state=PENDING, result=null, callback=(continuation of Deferred@1243862345 after retry RPC after error@198880940) -> (continuation of Deferred@1261886513 after retry RPC after error@1861431488) -> (continuation of Deferred@976103882 after retry RPC after error@1683967981) -> (continuation of Deferred@427151363 after retry RPC after error@1861719773) -> (continuation of Deferred@635212464 after retry RPC after error@1004700097) -> (continuation of Deferred@211856246 after retry RPC after error@1588827457) -> (continuation of Deferred@532399413 after retry RPC after error@821601162), errback=(continuation of Deferred@1243862345 after retry RPC after error@198880940) -> (continuation of Deferred@1261886513 after retry RPC after error@1861431488) -> (continuation of Deferred@976103882 after retry RPC after error@1683967981) -> (continuation of Deferred@427151363 after retry RPC after error@1861719773) -> (continuation of Deferred@635212464 after retry RPC after error@1004700097) -> (continuation of Deferred@211856246 after retry RPC after error@1588827457) -> (continuation of Deferred@532399413 after retry RPC after error@821601162)))
06:33:47.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@863246232(state=PENDING, result=null, callback=(continuation of Deferred@893422878 after retry RPC after error@687488238) -> (continuation of Deferred@1694735555 after retry RPC after error@1985519404) -> (continuation of Deferred@1203881092 after retry RPC after error@1051037150) -> (continuation of Deferred@1203370829 after retry RPC after error@1974119785) -> (continuation of Deferred@1015460540 after retry RPC after error@786367375) -> (continuation of Deferred@641001098 after retry RPC after error@106284736) -> (continuation of Deferred@256874468 after retry RPC after error@1507475193), errback=(continuation of Deferred@893422878 after retry RPC after error@687488238) -> (continuation of Deferred@1694735555 after retry RPC after error@1985519404) -> (continuation of Deferred@1203881092 after retry RPC after error@1051037150) -> (continuation of Deferred@1203370829 after retry RPC after error@1974119785) -> (continuation of Deferred@1015460540 after retry RPC after error@786367375) -> (continuation of Deferred@641001098 after retry RPC after error@106284736) -> (continuation of Deferred@256874468 after retry RPC after error@1507475193)))
06:33:47.672 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@863246232(state=PENDING, result=null, callback=(continuation of Deferred@893422878 after retry RPC after error@687488238) -> (continuation of Deferred@1694735555 after retry RPC after error@1985519404) -> (continuation of Deferred@1203881092 after retry RPC after error@1051037150) -> (continuation of Deferred@1203370829 after retry RPC after error@1974119785) -> (continuation of Deferred@1015460540 after retry RPC after error@786367375) -> (continuation of Deferred@641001098 after retry RPC after error@106284736) -> (continuation of Deferred@256874468 after retry RPC after error@1507475193), errback=(continuation of Deferred@893422878 after retry RPC after error@687488238) -> (continuation of Deferred@1694735555 after retry RPC after error@1985519404) -> (continuation of Deferred@1203881092 after retry RPC after error@1051037150) -> (continuation of Deferred@1203370829 after retry RPC after error@1974119785) -> (continuation of Deferred@1015460540 after retry RPC after error@786367375) -> (continuation of Deferred@641001098 after retry RPC after error@106284736) -> (continuation of Deferred@256874468 after retry RPC after error@1507475193)))
06:33:47.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.748 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.749 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.890 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.890 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.891 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@231411767(state=PENDING, result=null, callback=(continuation of Deferred@357051523 after retry RPC after error@737486265) -> (continuation of Deferred@1180323464 after retry RPC after error@323331292) -> (continuation of Deferred@1195228675 after retry RPC after error@1164411824) -> (continuation of Deferred@1373485978 after retry RPC after error@1870698041) -> (continuation of Deferred@1172828268 after retry RPC after error@1013223779) -> (continuation of Deferred@14285854 after retry RPC after error@1803780962) -> (continuation of Deferred@287140590 after retry RPC after error@166333991), errback=(continuation of Deferred@357051523 after retry RPC after error@737486265) -> (continuation of Deferred@1180323464 after retry RPC after error@323331292) -> (continuation of Deferred@1195228675 after retry RPC after error@1164411824) -> (continuation of Deferred@1373485978 after retry RPC after error@1870698041) -> (continuation of Deferred@1172828268 after retry RPC after error@1013223779) -> (continuation of Deferred@14285854 after retry RPC after error@1803780962) -> (continuation of Deferred@287140590 after retry RPC after error@166333991)))
06:33:47.892 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@231411767(state=PENDING, result=null, callback=(continuation of Deferred@357051523 after retry RPC after error@737486265) -> (continuation of Deferred@1180323464 after retry RPC after error@323331292) -> (continuation of Deferred@1195228675 after retry RPC after error@1164411824) -> (continuation of Deferred@1373485978 after retry RPC after error@1870698041) -> (continuation of Deferred@1172828268 after retry RPC after error@1013223779) -> (continuation of Deferred@14285854 after retry RPC after error@1803780962) -> (continuation of Deferred@287140590 after retry RPC after error@166333991), errback=(continuation of Deferred@357051523 after retry RPC after error@737486265) -> (continuation of Deferred@1180323464 after retry RPC after error@323331292) -> (continuation of Deferred@1195228675 after retry RPC after error@1164411824) -> (continuation of Deferred@1373485978 after retry RPC after error@1870698041) -> (continuation of Deferred@1172828268 after retry RPC after error@1013223779) -> (continuation of Deferred@14285854 after retry RPC after error@1803780962) -> (continuation of Deferred@287140590 after retry RPC after error@166333991)))
06:33:47.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:47.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:47.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:47.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:47.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:47.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.131 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@984617975(state=PENDING, result=null, callback=(continuation of Deferred@1413753268 after retry RPC after error@222699535) -> (continuation of Deferred@1771478614 after retry RPC after error@1401095992) -> (continuation of Deferred@654286197 after retry RPC after error@436574205) -> (continuation of Deferred@1195171525 after retry RPC after error@206416861) -> (continuation of Deferred@1443076884 after retry RPC after error@1620001878) -> (continuation of Deferred@1105486964 after retry RPC after error@538328278) -> (continuation of Deferred@1778895099 after retry RPC after error@1882233881), errback=(continuation of Deferred@1413753268 after retry RPC after error@222699535) -> (continuation of Deferred@1771478614 after retry RPC after error@1401095992) -> (continuation of Deferred@654286197 after retry RPC after error@436574205) -> (continuation of Deferred@1195171525 after retry RPC after error@206416861) -> (continuation of Deferred@1443076884 after retry RPC after error@1620001878) -> (continuation of Deferred@1105486964 after retry RPC after error@538328278) -> (continuation of Deferred@1778895099 after retry RPC after error@1882233881)))
06:33:48.131 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@984617975(state=PENDING, result=null, callback=(continuation of Deferred@1413753268 after retry RPC after error@222699535) -> (continuation of Deferred@1771478614 after retry RPC after error@1401095992) -> (continuation of Deferred@654286197 after retry RPC after error@436574205) -> (continuation of Deferred@1195171525 after retry RPC after error@206416861) -> (continuation of Deferred@1443076884 after retry RPC after error@1620001878) -> (continuation of Deferred@1105486964 after retry RPC after error@538328278) -> (continuation of Deferred@1778895099 after retry RPC after error@1882233881), errback=(continuation of Deferred@1413753268 after retry RPC after error@222699535) -> (continuation of Deferred@1771478614 after retry RPC after error@1401095992) -> (continuation of Deferred@654286197 after retry RPC after error@436574205) -> (continuation of Deferred@1195171525 after retry RPC after error@206416861) -> (continuation of Deferred@1443076884 after retry RPC after error@1620001878) -> (continuation of Deferred@1105486964 after retry RPC after error@538328278) -> (continuation of Deferred@1778895099 after retry RPC after error@1882233881)))
06:33:48.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.170 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.391 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1218592427(state=PENDING, result=null, callback=(continuation of Deferred@1538982948 after retry RPC after error@435173175) -> (continuation of Deferred@684606527 after retry RPC after error@1908437227) -> (continuation of Deferred@1918484691 after retry RPC after error@1778911264) -> (continuation of Deferred@817714914 after retry RPC after error@1116225334) -> (continuation of Deferred@2120954092 after retry RPC after error@1409838601) -> (continuation of Deferred@249051180 after retry RPC after error@229295950) -> (continuation of Deferred@719419975 after retry RPC after error@1024340276) -> (continuation of Deferred@1926789089 after retry RPC after error@689131075), errback=(continuation of Deferred@1538982948 after retry RPC after error@435173175) -> (continuation of Deferred@684606527 after retry RPC after error@1908437227) -> (continuation of Deferred@1918484691 after retry RPC after error@1778911264) -> (continuation of Deferred@817714914 after retry RPC after error@1116225334) -> (continuation of Deferred@2120954092 after retry RPC after error@1409838601) -> (continuation of Deferred@249051180 after retry RPC after error@229295950) -> (continuation of Deferred@719419975 after retry RPC after error@1024340276) -> (continuation of Deferred@1926789089 after retry RPC after error@689131075)))
06:33:48.391 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1218592427(state=PENDING, result=null, callback=(continuation of Deferred@1538982948 after retry RPC after error@435173175) -> (continuation of Deferred@684606527 after retry RPC after error@1908437227) -> (continuation of Deferred@1918484691 after retry RPC after error@1778911264) -> (continuation of Deferred@817714914 after retry RPC after error@1116225334) -> (continuation of Deferred@2120954092 after retry RPC after error@1409838601) -> (continuation of Deferred@249051180 after retry RPC after error@229295950) -> (continuation of Deferred@719419975 after retry RPC after error@1024340276) -> (continuation of Deferred@1926789089 after retry RPC after error@689131075), errback=(continuation of Deferred@1538982948 after retry RPC after error@435173175) -> (continuation of Deferred@684606527 after retry RPC after error@1908437227) -> (continuation of Deferred@1918484691 after retry RPC after error@1778911264) -> (continuation of Deferred@817714914 after retry RPC after error@1116225334) -> (continuation of Deferred@2120954092 after retry RPC after error@1409838601) -> (continuation of Deferred@249051180 after retry RPC after error@229295950) -> (continuation of Deferred@719419975 after retry RPC after error@1024340276) -> (continuation of Deferred@1926789089 after retry RPC after error@689131075)))
06:33:48.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.590 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@131408880(state=PENDING, result=null, callback=(continuation of Deferred@2112425261 after retry RPC after error@1078684096) -> (continuation of Deferred@419998513 after retry RPC after error@929878339) -> (continuation of Deferred@47788774 after retry RPC after error@432496148) -> (continuation of Deferred@1162489662 after retry RPC after error@352131101) -> (continuation of Deferred@1601717862 after retry RPC after error@2087486563) -> (continuation of Deferred@1758160655 after retry RPC after error@344635387) -> (continuation of Deferred@157063290 after retry RPC after error@285952686) -> (continuation of Deferred@575382687 after retry RPC after error@1673521730), errback=(continuation of Deferred@2112425261 after retry RPC after error@1078684096) -> (continuation of Deferred@419998513 after retry RPC after error@929878339) -> (continuation of Deferred@47788774 after retry RPC after error@432496148) -> (continuation of Deferred@1162489662 after retry RPC after error@352131101) -> (continuation of Deferred@1601717862 after retry RPC after error@2087486563) -> (continuation of Deferred@1758160655 after retry RPC after error@344635387) -> (continuation of Deferred@157063290 after retry RPC after error@285952686) -> (continuation of Deferred@575382687 after retry RPC after error@1673521730)))
06:33:48.590 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@131408880(state=PENDING, result=null, callback=(continuation of Deferred@2112425261 after retry RPC after error@1078684096) -> (continuation of Deferred@419998513 after retry RPC after error@929878339) -> (continuation of Deferred@47788774 after retry RPC after error@432496148) -> (continuation of Deferred@1162489662 after retry RPC after error@352131101) -> (continuation of Deferred@1601717862 after retry RPC after error@2087486563) -> (continuation of Deferred@1758160655 after retry RPC after error@344635387) -> (continuation of Deferred@157063290 after retry RPC after error@285952686) -> (continuation of Deferred@575382687 after retry RPC after error@1673521730), errback=(continuation of Deferred@2112425261 after retry RPC after error@1078684096) -> (continuation of Deferred@419998513 after retry RPC after error@929878339) -> (continuation of Deferred@47788774 after retry RPC after error@432496148) -> (continuation of Deferred@1162489662 after retry RPC after error@352131101) -> (continuation of Deferred@1601717862 after retry RPC after error@2087486563) -> (continuation of Deferred@1758160655 after retry RPC after error@344635387) -> (continuation of Deferred@157063290 after retry RPC after error@285952686) -> (continuation of Deferred@575382687 after retry RPC after error@1673521730)))
06:33:48.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.610 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.730 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.730 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.791 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2045287065(state=PENDING, result=null, callback=(continuation of Deferred@1865414827 after retry RPC after error@1515254518) -> (continuation of Deferred@903317427 after retry RPC after error@139915202) -> (continuation of Deferred@405583043 after retry RPC after error@1169568370) -> (continuation of Deferred@1289080680 after retry RPC after error@1942094287) -> (continuation of Deferred@238369240 after retry RPC after error@2061807566) -> (continuation of Deferred@405307765 after retry RPC after error@2064070598), errback=(continuation of Deferred@1865414827 after retry RPC after error@1515254518) -> (continuation of Deferred@903317427 after retry RPC after error@139915202) -> (continuation of Deferred@405583043 after retry RPC after error@1169568370) -> (continuation of Deferred@1289080680 after retry RPC after error@1942094287) -> (continuation of Deferred@238369240 after retry RPC after error@2061807566) -> (continuation of Deferred@405307765 after retry RPC after error@2064070598)))
06:33:48.791 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2045287065(state=PENDING, result=null, callback=(continuation of Deferred@1865414827 after retry RPC after error@1515254518) -> (continuation of Deferred@903317427 after retry RPC after error@139915202) -> (continuation of Deferred@405583043 after retry RPC after error@1169568370) -> (continuation of Deferred@1289080680 after retry RPC after error@1942094287) -> (continuation of Deferred@238369240 after retry RPC after error@2061807566) -> (continuation of Deferred@405307765 after retry RPC after error@2064070598), errback=(continuation of Deferred@1865414827 after retry RPC after error@1515254518) -> (continuation of Deferred@903317427 after retry RPC after error@139915202) -> (continuation of Deferred@405583043 after retry RPC after error@1169568370) -> (continuation of Deferred@1289080680 after retry RPC after error@1942094287) -> (continuation of Deferred@238369240 after retry RPC after error@2061807566) -> (continuation of Deferred@405307765 after retry RPC after error@2064070598)))
06:33:48.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.837 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:48.837777 25556 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:48.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:48.837961 25556 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:48.838 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:48.838510 25556 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:48.840 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:48.840655 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:48.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:48.840858 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:48.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:48.841024 25336 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:48.841 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:48.841317 25556 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:48.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:48.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:48.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:48.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:48.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:48.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.010 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@605378692(state=PENDING, result=null, callback=(continuation of Deferred@693099891 after retry RPC after error@1222213668) -> (continuation of Deferred@2015984377 after retry RPC after error@1995889664) -> (continuation of Deferred@553989096 after retry RPC after error@626609996) -> (continuation of Deferred@378804140 after retry RPC after error@1447219106) -> (continuation of Deferred@1590701297 after retry RPC after error@832318280) -> (continuation of Deferred@126055328 after retry RPC after error@1207735085) -> (continuation of Deferred@412387463 after retry RPC after error@1216493238), errback=(continuation of Deferred@693099891 after retry RPC after error@1222213668) -> (continuation of Deferred@2015984377 after retry RPC after error@1995889664) -> (continuation of Deferred@553989096 after retry RPC after error@626609996) -> (continuation of Deferred@378804140 after retry RPC after error@1447219106) -> (continuation of Deferred@1590701297 after retry RPC after error@832318280) -> (continuation of Deferred@126055328 after retry RPC after error@1207735085) -> (continuation of Deferred@412387463 after retry RPC after error@1216493238)))
06:33:49.011 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@605378692(state=PENDING, result=null, callback=(continuation of Deferred@693099891 after retry RPC after error@1222213668) -> (continuation of Deferred@2015984377 after retry RPC after error@1995889664) -> (continuation of Deferred@553989096 after retry RPC after error@626609996) -> (continuation of Deferred@378804140 after retry RPC after error@1447219106) -> (continuation of Deferred@1590701297 after retry RPC after error@832318280) -> (continuation of Deferred@126055328 after retry RPC after error@1207735085) -> (continuation of Deferred@412387463 after retry RPC after error@1216493238), errback=(continuation of Deferred@693099891 after retry RPC after error@1222213668) -> (continuation of Deferred@2015984377 after retry RPC after error@1995889664) -> (continuation of Deferred@553989096 after retry RPC after error@626609996) -> (continuation of Deferred@378804140 after retry RPC after error@1447219106) -> (continuation of Deferred@1590701297 after retry RPC after error@832318280) -> (continuation of Deferred@126055328 after retry RPC after error@1207735085) -> (continuation of Deferred@412387463 after retry RPC after error@1216493238)))
06:33:49.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.251 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@322477376(state=PENDING, result=null, callback=(continuation of Deferred@1414080732 after retry RPC after error@1261672757) -> (continuation of Deferred@1719700200 after retry RPC after error@830851557) -> (continuation of Deferred@739711169 after retry RPC after error@2067453511) -> (continuation of Deferred@428856322 after retry RPC after error@115010615) -> (continuation of Deferred@312421693 after retry RPC after error@1060788679) -> (continuation of Deferred@1492297190 after retry RPC after error@160973211) -> (continuation of Deferred@1644520272 after retry RPC after error@261941942), errback=(continuation of Deferred@1414080732 after retry RPC after error@1261672757) -> (continuation of Deferred@1719700200 after retry RPC after error@830851557) -> (continuation of Deferred@739711169 after retry RPC after error@2067453511) -> (continuation of Deferred@428856322 after retry RPC after error@115010615) -> (continuation of Deferred@312421693 after retry RPC after error@1060788679) -> (continuation of Deferred@1492297190 after retry RPC after error@160973211) -> (continuation of Deferred@1644520272 after retry RPC after error@261941942)))
06:33:49.251 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@322477376(state=PENDING, result=null, callback=(continuation of Deferred@1414080732 after retry RPC after error@1261672757) -> (continuation of Deferred@1719700200 after retry RPC after error@830851557) -> (continuation of Deferred@739711169 after retry RPC after error@2067453511) -> (continuation of Deferred@428856322 after retry RPC after error@115010615) -> (continuation of Deferred@312421693 after retry RPC after error@1060788679) -> (continuation of Deferred@1492297190 after retry RPC after error@160973211) -> (continuation of Deferred@1644520272 after retry RPC after error@261941942), errback=(continuation of Deferred@1414080732 after retry RPC after error@1261672757) -> (continuation of Deferred@1719700200 after retry RPC after error@830851557) -> (continuation of Deferred@739711169 after retry RPC after error@2067453511) -> (continuation of Deferred@428856322 after retry RPC after error@115010615) -> (continuation of Deferred@312421693 after retry RPC after error@1060788679) -> (continuation of Deferred@1492297190 after retry RPC after error@160973211) -> (continuation of Deferred@1644520272 after retry RPC after error@261941942)))
06:33:49.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.431 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1811962708(state=PENDING, result=null, callback=(continuation of Deferred@1196565654 after retry RPC after error@1492277297) -> (continuation of Deferred@869383395 after retry RPC after error@1575324470) -> (continuation of Deferred@189096584 after retry RPC after error@1094488065) -> (continuation of Deferred@2134121057 after retry RPC after error@251232906) -> (continuation of Deferred@772333823 after retry RPC after error@194101671) -> (continuation of Deferred@1411986854 after retry RPC after error@377880699), errback=(continuation of Deferred@1196565654 after retry RPC after error@1492277297) -> (continuation of Deferred@869383395 after retry RPC after error@1575324470) -> (continuation of Deferred@189096584 after retry RPC after error@1094488065) -> (continuation of Deferred@2134121057 after retry RPC after error@251232906) -> (continuation of Deferred@772333823 after retry RPC after error@194101671) -> (continuation of Deferred@1411986854 after retry RPC after error@377880699)))
06:33:49.431 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1811962708(state=PENDING, result=null, callback=(continuation of Deferred@1196565654 after retry RPC after error@1492277297) -> (continuation of Deferred@869383395 after retry RPC after error@1575324470) -> (continuation of Deferred@189096584 after retry RPC after error@1094488065) -> (continuation of Deferred@2134121057 after retry RPC after error@251232906) -> (continuation of Deferred@772333823 after retry RPC after error@194101671) -> (continuation of Deferred@1411986854 after retry RPC after error@377880699), errback=(continuation of Deferred@1196565654 after retry RPC after error@1492277297) -> (continuation of Deferred@869383395 after retry RPC after error@1575324470) -> (continuation of Deferred@189096584 after retry RPC after error@1094488065) -> (continuation of Deferred@2134121057 after retry RPC after error@251232906) -> (continuation of Deferred@772333823 after retry RPC after error@194101671) -> (continuation of Deferred@1411986854 after retry RPC after error@377880699)))
06:33:49.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.590 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@930135570(state=PENDING, result=null, callback=(continuation of Deferred@608059693 after retry RPC after error@365538218) -> (continuation of Deferred@211585032 after retry RPC after error@1051600807) -> (continuation of Deferred@1750019702 after retry RPC after error@805522478) -> (continuation of Deferred@1480019694 after retry RPC after error@392939856) -> (continuation of Deferred@105281877 after retry RPC after error@747517839) -> (continuation of Deferred@1311526094 after retry RPC after error@130301202) -> (continuation of Deferred@855466094 after retry RPC after error@1871842289), errback=(continuation of Deferred@608059693 after retry RPC after error@365538218) -> (continuation of Deferred@211585032 after retry RPC after error@1051600807) -> (continuation of Deferred@1750019702 after retry RPC after error@805522478) -> (continuation of Deferred@1480019694 after retry RPC after error@392939856) -> (continuation of Deferred@105281877 after retry RPC after error@747517839) -> (continuation of Deferred@1311526094 after retry RPC after error@130301202) -> (continuation of Deferred@855466094 after retry RPC after error@1871842289)))
06:33:49.691 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@930135570(state=PENDING, result=null, callback=(continuation of Deferred@608059693 after retry RPC after error@365538218) -> (continuation of Deferred@211585032 after retry RPC after error@1051600807) -> (continuation of Deferred@1750019702 after retry RPC after error@805522478) -> (continuation of Deferred@1480019694 after retry RPC after error@392939856) -> (continuation of Deferred@105281877 after retry RPC after error@747517839) -> (continuation of Deferred@1311526094 after retry RPC after error@130301202) -> (continuation of Deferred@855466094 after retry RPC after error@1871842289), errback=(continuation of Deferred@608059693 after retry RPC after error@365538218) -> (continuation of Deferred@211585032 after retry RPC after error@1051600807) -> (continuation of Deferred@1750019702 after retry RPC after error@805522478) -> (continuation of Deferred@1480019694 after retry RPC after error@392939856) -> (continuation of Deferred@105281877 after retry RPC after error@747517839) -> (continuation of Deferred@1311526094 after retry RPC after error@130301202) -> (continuation of Deferred@855466094 after retry RPC after error@1871842289)))
06:33:49.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.730 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.870 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.910 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.910 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@826410680(state=PENDING, result=null, callback=(continuation of Deferred@628767759 after retry RPC after error@1656246284) -> (continuation of Deferred@1917711768 after retry RPC after error@1394301615) -> (continuation of Deferred@670408327 after retry RPC after error@133005893) -> (continuation of Deferred@445728277 after retry RPC after error@324446666) -> (continuation of Deferred@408685697 after retry RPC after error@1934121573) -> (continuation of Deferred@1668877303 after retry RPC after error@1170728619) -> (continuation of Deferred@1023918742 after retry RPC after error@204287009) -> (continuation of Deferred@664840416 after retry RPC after error@1571348481), errback=(continuation of Deferred@628767759 after retry RPC after error@1656246284) -> (continuation of Deferred@1917711768 after retry RPC after error@1394301615) -> (continuation of Deferred@670408327 after retry RPC after error@133005893) -> (continuation of Deferred@445728277 after retry RPC after error@324446666) -> (continuation of Deferred@408685697 after retry RPC after error@1934121573) -> (continuation of Deferred@1668877303 after retry RPC after error@1170728619) -> (continuation of Deferred@1023918742 after retry RPC after error@204287009) -> (continuation of Deferred@664840416 after retry RPC after error@1571348481)))
06:33:49.952 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@826410680(state=PENDING, result=null, callback=(continuation of Deferred@628767759 after retry RPC after error@1656246284) -> (continuation of Deferred@1917711768 after retry RPC after error@1394301615) -> (continuation of Deferred@670408327 after retry RPC after error@133005893) -> (continuation of Deferred@445728277 after retry RPC after error@324446666) -> (continuation of Deferred@408685697 after retry RPC after error@1934121573) -> (continuation of Deferred@1668877303 after retry RPC after error@1170728619) -> (continuation of Deferred@1023918742 after retry RPC after error@204287009) -> (continuation of Deferred@664840416 after retry RPC after error@1571348481), errback=(continuation of Deferred@628767759 after retry RPC after error@1656246284) -> (continuation of Deferred@1917711768 after retry RPC after error@1394301615) -> (continuation of Deferred@670408327 after retry RPC after error@133005893) -> (continuation of Deferred@445728277 after retry RPC after error@324446666) -> (continuation of Deferred@408685697 after retry RPC after error@1934121573) -> (continuation of Deferred@1668877303 after retry RPC after error@1170728619) -> (continuation of Deferred@1023918742 after retry RPC after error@204287009) -> (continuation of Deferred@664840416 after retry RPC after error@1571348481)))
06:33:49.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:49.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:49.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:49.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:49.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:49.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:50.015537 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:50.018 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:50.017946 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:50.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:50.019331 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:33:50.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1480480114(state=PENDING, result=null, callback=(continuation of Deferred@908676113 after retry RPC after error@1531085466) -> (continuation of Deferred@1918587155 after retry RPC after error@1482952687) -> (continuation of Deferred@609327414 after retry RPC after error@1799915892) -> (continuation of Deferred@1127452359 after retry RPC after error@1003547193) -> (continuation of Deferred@325078108 after retry RPC after error@1020032109) -> (continuation of Deferred@1666613381 after retry RPC after error@1609252852), errback=(continuation of Deferred@908676113 after retry RPC after error@1531085466) -> (continuation of Deferred@1918587155 after retry RPC after error@1482952687) -> (continuation of Deferred@609327414 after retry RPC after error@1799915892) -> (continuation of Deferred@1127452359 after retry RPC after error@1003547193) -> (continuation of Deferred@325078108 after retry RPC after error@1020032109) -> (continuation of Deferred@1666613381 after retry RPC after error@1609252852)))
06:33:50.111 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1480480114(state=PENDING, result=null, callback=(continuation of Deferred@908676113 after retry RPC after error@1531085466) -> (continuation of Deferred@1918587155 after retry RPC after error@1482952687) -> (continuation of Deferred@609327414 after retry RPC after error@1799915892) -> (continuation of Deferred@1127452359 after retry RPC after error@1003547193) -> (continuation of Deferred@325078108 after retry RPC after error@1020032109) -> (continuation of Deferred@1666613381 after retry RPC after error@1609252852), errback=(continuation of Deferred@908676113 after retry RPC after error@1531085466) -> (continuation of Deferred@1918587155 after retry RPC after error@1482952687) -> (continuation of Deferred@609327414 after retry RPC after error@1799915892) -> (continuation of Deferred@1127452359 after retry RPC after error@1003547193) -> (continuation of Deferred@325078108 after retry RPC after error@1020032109) -> (continuation of Deferred@1666613381 after retry RPC after error@1609252852)))
06:33:50.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.150 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1201487288(state=PENDING, result=null, callback=(continuation of Deferred@1616480024 after retry RPC after error@211219093) -> (continuation of Deferred@398665986 after retry RPC after error@958244543) -> (continuation of Deferred@357603267 after retry RPC after error@778689958) -> (continuation of Deferred@552368720 after retry RPC after error@1955600556) -> (continuation of Deferred@1701799794 after retry RPC after error@84611732) -> (continuation of Deferred@1190581919 after retry RPC after error@853009301) -> (continuation of Deferred@740120138 after retry RPC after error@1910472431), errback=(continuation of Deferred@1616480024 after retry RPC after error@211219093) -> (continuation of Deferred@398665986 after retry RPC after error@958244543) -> (continuation of Deferred@357603267 after retry RPC after error@778689958) -> (continuation of Deferred@552368720 after retry RPC after error@1955600556) -> (continuation of Deferred@1701799794 after retry RPC after error@84611732) -> (continuation of Deferred@1190581919 after retry RPC after error@853009301) -> (continuation of Deferred@740120138 after retry RPC after error@1910472431)))
06:33:50.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1201487288(state=PENDING, result=null, callback=(continuation of Deferred@1616480024 after retry RPC after error@211219093) -> (continuation of Deferred@398665986 after retry RPC after error@958244543) -> (continuation of Deferred@357603267 after retry RPC after error@778689958) -> (continuation of Deferred@552368720 after retry RPC after error@1955600556) -> (continuation of Deferred@1701799794 after retry RPC after error@84611732) -> (continuation of Deferred@1190581919 after retry RPC after error@853009301) -> (continuation of Deferred@740120138 after retry RPC after error@1910472431), errback=(continuation of Deferred@1616480024 after retry RPC after error@211219093) -> (continuation of Deferred@398665986 after retry RPC after error@958244543) -> (continuation of Deferred@357603267 after retry RPC after error@778689958) -> (continuation of Deferred@552368720 after retry RPC after error@1955600556) -> (continuation of Deferred@1701799794 after retry RPC after error@84611732) -> (continuation of Deferred@1190581919 after retry RPC after error@853009301) -> (continuation of Deferred@740120138 after retry RPC after error@1910472431)))
06:33:50.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.308 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.308 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.550 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.550 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.552 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1232246838(state=PENDING, result=null, callback=(continuation of Deferred@561070616 after retry RPC after error@2000382396) -> (continuation of Deferred@1556714720 after retry RPC after error@27726951) -> (continuation of Deferred@720238537 after retry RPC after error@112659873) -> (continuation of Deferred@373946115 after retry RPC after error@796183937) -> (continuation of Deferred@1784820761 after retry RPC after error@1409742577) -> (continuation of Deferred@221176477 after retry RPC after error@71249532) -> (continuation of Deferred@2131099678 after retry RPC after error@483704945), errback=(continuation of Deferred@561070616 after retry RPC after error@2000382396) -> (continuation of Deferred@1556714720 after retry RPC after error@27726951) -> (continuation of Deferred@720238537 after retry RPC after error@112659873) -> (continuation of Deferred@373946115 after retry RPC after error@796183937) -> (continuation of Deferred@1784820761 after retry RPC after error@1409742577) -> (continuation of Deferred@221176477 after retry RPC after error@71249532) -> (continuation of Deferred@2131099678 after retry RPC after error@483704945)))
06:33:50.553 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1232246838(state=PENDING, result=null, callback=(continuation of Deferred@561070616 after retry RPC after error@2000382396) -> (continuation of Deferred@1556714720 after retry RPC after error@27726951) -> (continuation of Deferred@720238537 after retry RPC after error@112659873) -> (continuation of Deferred@373946115 after retry RPC after error@796183937) -> (continuation of Deferred@1784820761 after retry RPC after error@1409742577) -> (continuation of Deferred@221176477 after retry RPC after error@71249532) -> (continuation of Deferred@2131099678 after retry RPC after error@483704945), errback=(continuation of Deferred@561070616 after retry RPC after error@2000382396) -> (continuation of Deferred@1556714720 after retry RPC after error@27726951) -> (continuation of Deferred@720238537 after retry RPC after error@112659873) -> (continuation of Deferred@373946115 after retry RPC after error@796183937) -> (continuation of Deferred@1784820761 after retry RPC after error@1409742577) -> (continuation of Deferred@221176477 after retry RPC after error@71249532) -> (continuation of Deferred@2131099678 after retry RPC after error@483704945)))
06:33:50.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.731 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1775560516(state=PENDING, result=null, callback=(continuation of Deferred@1456210843 after retry RPC after error@1598988249) -> (continuation of Deferred@527018098 after retry RPC after error@1363771932) -> (continuation of Deferred@1739559797 after retry RPC after error@1286891534) -> (continuation of Deferred@516715474 after retry RPC after error@50875253) -> (continuation of Deferred@413619290 after retry RPC after error@2008857432) -> (continuation of Deferred@177411230 after retry RPC after error@1258403162), errback=(continuation of Deferred@1456210843 after retry RPC after error@1598988249) -> (continuation of Deferred@527018098 after retry RPC after error@1363771932) -> (continuation of Deferred@1739559797 after retry RPC after error@1286891534) -> (continuation of Deferred@516715474 after retry RPC after error@50875253) -> (continuation of Deferred@413619290 after retry RPC after error@2008857432) -> (continuation of Deferred@177411230 after retry RPC after error@1258403162)))
06:33:50.731 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1775560516(state=PENDING, result=null, callback=(continuation of Deferred@1456210843 after retry RPC after error@1598988249) -> (continuation of Deferred@527018098 after retry RPC after error@1363771932) -> (continuation of Deferred@1739559797 after retry RPC after error@1286891534) -> (continuation of Deferred@516715474 after retry RPC after error@50875253) -> (continuation of Deferred@413619290 after retry RPC after error@2008857432) -> (continuation of Deferred@177411230 after retry RPC after error@1258403162), errback=(continuation of Deferred@1456210843 after retry RPC after error@1598988249) -> (continuation of Deferred@527018098 after retry RPC after error@1363771932) -> (continuation of Deferred@1739559797 after retry RPC after error@1286891534) -> (continuation of Deferred@516715474 after retry RPC after error@50875253) -> (continuation of Deferred@413619290 after retry RPC after error@2008857432) -> (continuation of Deferred@177411230 after retry RPC after error@1258403162)))
06:33:50.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@856439(state=PENDING, result=null, callback=(continuation of Deferred@545562597 after retry RPC after error@652330863) -> (continuation of Deferred@238675369 after retry RPC after error@1717029808) -> (continuation of Deferred@1655184168 after retry RPC after error@1255830189) -> (continuation of Deferred@185586418 after retry RPC after error@1398641256) -> (continuation of Deferred@1896788592 after retry RPC after error@1750657510) -> (continuation of Deferred@498417245 after retry RPC after error@583278929), errback=(continuation of Deferred@545562597 after retry RPC after error@652330863) -> (continuation of Deferred@238675369 after retry RPC after error@1717029808) -> (continuation of Deferred@1655184168 after retry RPC after error@1255830189) -> (continuation of Deferred@185586418 after retry RPC after error@1398641256) -> (continuation of Deferred@1896788592 after retry RPC after error@1750657510) -> (continuation of Deferred@498417245 after retry RPC after error@583278929)))
06:33:50.952 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@856439(state=PENDING, result=null, callback=(continuation of Deferred@545562597 after retry RPC after error@652330863) -> (continuation of Deferred@238675369 after retry RPC after error@1717029808) -> (continuation of Deferred@1655184168 after retry RPC after error@1255830189) -> (continuation of Deferred@185586418 after retry RPC after error@1398641256) -> (continuation of Deferred@1896788592 after retry RPC after error@1750657510) -> (continuation of Deferred@498417245 after retry RPC after error@583278929), errback=(continuation of Deferred@545562597 after retry RPC after error@652330863) -> (continuation of Deferred@238675369 after retry RPC after error@1717029808) -> (continuation of Deferred@1655184168 after retry RPC after error@1255830189) -> (continuation of Deferred@185586418 after retry RPC after error@1398641256) -> (continuation of Deferred@1896788592 after retry RPC after error@1750657510) -> (continuation of Deferred@498417245 after retry RPC after error@583278929)))
06:33:50.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:50.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:50.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:50.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:50.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:50.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@2044823347(state=PENDING, result=null, callback=(continuation of Deferred@1286635 after retry RPC after error@1465932440) -> (continuation of Deferred@226509802 after retry RPC after error@2024641910) -> (continuation of Deferred@1697415025 after retry RPC after error@818761174) -> (continuation of Deferred@1071503158 after retry RPC after error@735048606) -> (continuation of Deferred@765384289 after retry RPC after error@2011667961) -> (continuation of Deferred@642945108 after retry RPC after error@1918227459) -> (continuation of Deferred@690174454 after retry RPC after error@1650487678), errback=(continuation of Deferred@1286635 after retry RPC after error@1465932440) -> (continuation of Deferred@226509802 after retry RPC after error@2024641910) -> (continuation of Deferred@1697415025 after retry RPC after error@818761174) -> (continuation of Deferred@1071503158 after retry RPC after error@735048606) -> (continuation of Deferred@765384289 after retry RPC after error@2011667961) -> (continuation of Deferred@642945108 after retry RPC after error@1918227459) -> (continuation of Deferred@690174454 after retry RPC after error@1650487678)))
06:33:51.151 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@2044823347(state=PENDING, result=null, callback=(continuation of Deferred@1286635 after retry RPC after error@1465932440) -> (continuation of Deferred@226509802 after retry RPC after error@2024641910) -> (continuation of Deferred@1697415025 after retry RPC after error@818761174) -> (continuation of Deferred@1071503158 after retry RPC after error@735048606) -> (continuation of Deferred@765384289 after retry RPC after error@2011667961) -> (continuation of Deferred@642945108 after retry RPC after error@1918227459) -> (continuation of Deferred@690174454 after retry RPC after error@1650487678), errback=(continuation of Deferred@1286635 after retry RPC after error@1465932440) -> (continuation of Deferred@226509802 after retry RPC after error@2024641910) -> (continuation of Deferred@1697415025 after retry RPC after error@818761174) -> (continuation of Deferred@1071503158 after retry RPC after error@735048606) -> (continuation of Deferred@765384289 after retry RPC after error@2011667961) -> (continuation of Deferred@642945108 after retry RPC after error@1918227459) -> (continuation of Deferred@690174454 after retry RPC after error@1650487678)))
06:33:51.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.190 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.210 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.251 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.251 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.251 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.251 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.273 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.273 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.273 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.273 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.273 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.273 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.273 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.274 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.274 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.350 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1496052358(state=PENDING, result=null, callback=(continuation of Deferred@818956646 after retry RPC after error@890025977) -> (continuation of Deferred@1242566380 after retry RPC after error@1277786858) -> (continuation of Deferred@2122350960 after retry RPC after error@786649638) -> (continuation of Deferred@1682857592 after retry RPC after error@1664587313) -> (continuation of Deferred@424022054 after retry RPC after error@754979658) -> (continuation of Deferred@992683276 after retry RPC after error@1685240390), errback=(continuation of Deferred@818956646 after retry RPC after error@890025977) -> (continuation of Deferred@1242566380 after retry RPC after error@1277786858) -> (continuation of Deferred@2122350960 after retry RPC after error@786649638) -> (continuation of Deferred@1682857592 after retry RPC after error@1664587313) -> (continuation of Deferred@424022054 after retry RPC after error@754979658) -> (continuation of Deferred@992683276 after retry RPC after error@1685240390)))
06:33:51.351 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [106ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1496052358(state=PENDING, result=null, callback=(continuation of Deferred@818956646 after retry RPC after error@890025977) -> (continuation of Deferred@1242566380 after retry RPC after error@1277786858) -> (continuation of Deferred@2122350960 after retry RPC after error@786649638) -> (continuation of Deferred@1682857592 after retry RPC after error@1664587313) -> (continuation of Deferred@424022054 after retry RPC after error@754979658) -> (continuation of Deferred@992683276 after retry RPC after error@1685240390), errback=(continuation of Deferred@818956646 after retry RPC after error@890025977) -> (continuation of Deferred@1242566380 after retry RPC after error@1277786858) -> (continuation of Deferred@2122350960 after retry RPC after error@786649638) -> (continuation of Deferred@1682857592 after retry RPC after error@1664587313) -> (continuation of Deferred@424022054 after retry RPC after error@754979658) -> (continuation of Deferred@992683276 after retry RPC after error@1685240390)))
06:33:51.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.389 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.588 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.589 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.590 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1666465337(state=PENDING, result=null, callback=(continuation of Deferred@995979405 after retry RPC after error@1328201442) -> (continuation of Deferred@1684323996 after retry RPC after error@646181343) -> (continuation of Deferred@1988764826 after retry RPC after error@1825396999) -> (continuation of Deferred@765233756 after retry RPC after error@338201401) -> (continuation of Deferred@866239442 after retry RPC after error@891466176) -> (continuation of Deferred@2082804562 after retry RPC after error@746864300) -> (continuation of Deferred@294049725 after retry RPC after error@822794492) -> (continuation of Deferred@1700917772 after retry RPC after error@1746784594), errback=(continuation of Deferred@995979405 after retry RPC after error@1328201442) -> (continuation of Deferred@1684323996 after retry RPC after error@646181343) -> (continuation of Deferred@1988764826 after retry RPC after error@1825396999) -> (continuation of Deferred@765233756 after retry RPC after error@338201401) -> (continuation of Deferred@866239442 after retry RPC after error@891466176) -> (continuation of Deferred@2082804562 after retry RPC after error@746864300) -> (continuation of Deferred@294049725 after retry RPC after error@822794492) -> (continuation of Deferred@1700917772 after retry RPC after error@1746784594)))
06:33:51.590 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1666465337(state=PENDING, result=null, callback=(continuation of Deferred@995979405 after retry RPC after error@1328201442) -> (continuation of Deferred@1684323996 after retry RPC after error@646181343) -> (continuation of Deferred@1988764826 after retry RPC after error@1825396999) -> (continuation of Deferred@765233756 after retry RPC after error@338201401) -> (continuation of Deferred@866239442 after retry RPC after error@891466176) -> (continuation of Deferred@2082804562 after retry RPC after error@746864300) -> (continuation of Deferred@294049725 after retry RPC after error@822794492) -> (continuation of Deferred@1700917772 after retry RPC after error@1746784594), errback=(continuation of Deferred@995979405 after retry RPC after error@1328201442) -> (continuation of Deferred@1684323996 after retry RPC after error@646181343) -> (continuation of Deferred@1988764826 after retry RPC after error@1825396999) -> (continuation of Deferred@765233756 after retry RPC after error@338201401) -> (continuation of Deferred@866239442 after retry RPC after error@891466176) -> (continuation of Deferred@2082804562 after retry RPC after error@746864300) -> (continuation of Deferred@294049725 after retry RPC after error@822794492) -> (continuation of Deferred@1700917772 after retry RPC after error@1746784594)))
06:33:51.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.869 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@219510631(state=PENDING, result=null, callback=(continuation of Deferred@1205787382 after retry RPC after error@1210851793) -> (continuation of Deferred@158142651 after retry RPC after error@795209918) -> (continuation of Deferred@509445838 after retry RPC after error@936779389) -> (continuation of Deferred@1449081076 after retry RPC after error@1447023233) -> (continuation of Deferred@606492030 after retry RPC after error@595890801) -> (continuation of Deferred@213096045 after retry RPC after error@607188640) -> (continuation of Deferred@841730813 after retry RPC after error@893108691), errback=(continuation of Deferred@1205787382 after retry RPC after error@1210851793) -> (continuation of Deferred@158142651 after retry RPC after error@795209918) -> (continuation of Deferred@509445838 after retry RPC after error@936779389) -> (continuation of Deferred@1449081076 after retry RPC after error@1447023233) -> (continuation of Deferred@606492030 after retry RPC after error@595890801) -> (continuation of Deferred@213096045 after retry RPC after error@607188640) -> (continuation of Deferred@841730813 after retry RPC after error@893108691)))
06:33:51.869 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:51.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.948 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.948 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.948 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.948 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.949 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:51.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:51.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:51.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:51.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:51.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.110 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@267990452(state=PENDING, result=null, callback=(continuation of Deferred@2136449279 after retry RPC after error@742529976) -> (continuation of Deferred@1730620623 after retry RPC after error@565571850) -> (continuation of Deferred@1671953819 after retry RPC after error@1594144140) -> (continuation of Deferred@26979319 after retry RPC after error@254289775) -> (continuation of Deferred@1396656180 after retry RPC after error@703414026) -> (continuation of Deferred@1970662867 after retry RPC after error@992522061) -> (continuation of Deferred@318717684 after retry RPC after error@1386895054), errback=(continuation of Deferred@2136449279 after retry RPC after error@742529976) -> (continuation of Deferred@1730620623 after retry RPC after error@565571850) -> (continuation of Deferred@1671953819 after retry RPC after error@1594144140) -> (continuation of Deferred@26979319 after retry RPC after error@254289775) -> (continuation of Deferred@1396656180 after retry RPC after error@703414026) -> (continuation of Deferred@1970662867 after retry RPC after error@992522061) -> (continuation of Deferred@318717684 after retry RPC after error@1386895054)))
06:33:52.111 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@267990452(state=PENDING, result=null, callback=(continuation of Deferred@2136449279 after retry RPC after error@742529976) -> (continuation of Deferred@1730620623 after retry RPC after error@565571850) -> (continuation of Deferred@1671953819 after retry RPC after error@1594144140) -> (continuation of Deferred@26979319 after retry RPC after error@254289775) -> (continuation of Deferred@1396656180 after retry RPC after error@703414026) -> (continuation of Deferred@1970662867 after retry RPC after error@992522061) -> (continuation of Deferred@318717684 after retry RPC after error@1386895054), errback=(continuation of Deferred@2136449279 after retry RPC after error@742529976) -> (continuation of Deferred@1730620623 after retry RPC after error@565571850) -> (continuation of Deferred@1671953819 after retry RPC after error@1594144140) -> (continuation of Deferred@26979319 after retry RPC after error@254289775) -> (continuation of Deferred@1396656180 after retry RPC after error@703414026) -> (continuation of Deferred@1970662867 after retry RPC after error@992522061) -> (continuation of Deferred@318717684 after retry RPC after error@1386895054)))
06:33:52.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.170 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.170 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.170 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.170 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.188 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.251 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.251 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@434068640(state=PENDING, result=null, callback=(continuation of Deferred@1294451448 after retry RPC after error@1465385605) -> (continuation of Deferred@888123136 after retry RPC after error@1471598433) -> (continuation of Deferred@620099437 after retry RPC after error@867079516) -> (continuation of Deferred@2054204294 after retry RPC after error@459058798) -> (continuation of Deferred@278490359 after retry RPC after error@1473912781) -> (continuation of Deferred@671719979 after retry RPC after error@1448625818), errback=(continuation of Deferred@1294451448 after retry RPC after error@1465385605) -> (continuation of Deferred@888123136 after retry RPC after error@1471598433) -> (continuation of Deferred@620099437 after retry RPC after error@867079516) -> (continuation of Deferred@2054204294 after retry RPC after error@459058798) -> (continuation of Deferred@278490359 after retry RPC after error@1473912781) -> (continuation of Deferred@671719979 after retry RPC after error@1448625818)))
06:33:52.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@434068640(state=PENDING, result=null, callback=(continuation of Deferred@1294451448 after retry RPC after error@1465385605) -> (continuation of Deferred@888123136 after retry RPC after error@1471598433) -> (continuation of Deferred@620099437 after retry RPC after error@867079516) -> (continuation of Deferred@2054204294 after retry RPC after error@459058798) -> (continuation of Deferred@278490359 after retry RPC after error@1473912781) -> (continuation of Deferred@671719979 after retry RPC after error@1448625818), errback=(continuation of Deferred@1294451448 after retry RPC after error@1465385605) -> (continuation of Deferred@888123136 after retry RPC after error@1471598433) -> (continuation of Deferred@620099437 after retry RPC after error@867079516) -> (continuation of Deferred@2054204294 after retry RPC after error@459058798) -> (continuation of Deferred@278490359 after retry RPC after error@1473912781) -> (continuation of Deferred@671719979 after retry RPC after error@1448625818)))
06:33:52.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.316 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:52.316783 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:33:52.317 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:52.316978 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:52.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1398763302(state=PENDING, result=null, callback=(continuation of Deferred@1133111002 after retry RPC after error@559533602) -> (continuation of Deferred@1039427774 after retry RPC after error@368711289) -> (continuation of Deferred@424254762 after retry RPC after error@1980655871) -> (continuation of Deferred@1747013400 after retry RPC after error@766363067) -> (continuation of Deferred@2019077609 after retry RPC after error@2098494349) -> (continuation of Deferred@531604152 after retry RPC after error@1550593259) -> (continuation of Deferred@1642521962 after retry RPC after error@1072706248), errback=(continuation of Deferred@1133111002 after retry RPC after error@559533602) -> (continuation of Deferred@1039427774 after retry RPC after error@368711289) -> (continuation of Deferred@424254762 after retry RPC after error@1980655871) -> (continuation of Deferred@1747013400 after retry RPC after error@766363067) -> (continuation of Deferred@2019077609 after retry RPC after error@2098494349) -> (continuation of Deferred@531604152 after retry RPC after error@1550593259) -> (continuation of Deferred@1642521962 after retry RPC after error@1072706248)))
06:33:52.511 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1398763302(state=PENDING, result=null, callback=(continuation of Deferred@1133111002 after retry RPC after error@559533602) -> (continuation of Deferred@1039427774 after retry RPC after error@368711289) -> (continuation of Deferred@424254762 after retry RPC after error@1980655871) -> (continuation of Deferred@1747013400 after retry RPC after error@766363067) -> (continuation of Deferred@2019077609 after retry RPC after error@2098494349) -> (continuation of Deferred@531604152 after retry RPC after error@1550593259) -> (continuation of Deferred@1642521962 after retry RPC after error@1072706248), errback=(continuation of Deferred@1133111002 after retry RPC after error@559533602) -> (continuation of Deferred@1039427774 after retry RPC after error@368711289) -> (continuation of Deferred@424254762 after retry RPC after error@1980655871) -> (continuation of Deferred@1747013400 after retry RPC after error@766363067) -> (continuation of Deferred@2019077609 after retry RPC after error@2098494349) -> (continuation of Deferred@531604152 after retry RPC after error@1550593259) -> (continuation of Deferred@1642521962 after retry RPC after error@1072706248)))
06:33:52.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.751 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1748226426(state=PENDING, result=null, callback=(continuation of Deferred@2004288850 after retry RPC after error@1837219775) -> (continuation of Deferred@1841024171 after retry RPC after error@469067879) -> (continuation of Deferred@1416078302 after retry RPC after error@888898218) -> (continuation of Deferred@118935050 after retry RPC after error@923805919) -> (continuation of Deferred@1585452208 after retry RPC after error@914760472) -> (continuation of Deferred@122801500 after retry RPC after error@825314233) -> (continuation of Deferred@1867185339 after retry RPC after error@1750305961), errback=(continuation of Deferred@2004288850 after retry RPC after error@1837219775) -> (continuation of Deferred@1841024171 after retry RPC after error@469067879) -> (continuation of Deferred@1416078302 after retry RPC after error@888898218) -> (continuation of Deferred@118935050 after retry RPC after error@923805919) -> (continuation of Deferred@1585452208 after retry RPC after error@914760472) -> (continuation of Deferred@122801500 after retry RPC after error@825314233) -> (continuation of Deferred@1867185339 after retry RPC after error@1750305961)))
06:33:52.751 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1748226426(state=PENDING, result=null, callback=(continuation of Deferred@2004288850 after retry RPC after error@1837219775) -> (continuation of Deferred@1841024171 after retry RPC after error@469067879) -> (continuation of Deferred@1416078302 after retry RPC after error@888898218) -> (continuation of Deferred@118935050 after retry RPC after error@923805919) -> (continuation of Deferred@1585452208 after retry RPC after error@914760472) -> (continuation of Deferred@122801500 after retry RPC after error@825314233) -> (continuation of Deferred@1867185339 after retry RPC after error@1750305961), errback=(continuation of Deferred@2004288850 after retry RPC after error@1837219775) -> (continuation of Deferred@1841024171 after retry RPC after error@469067879) -> (continuation of Deferred@1416078302 after retry RPC after error@888898218) -> (continuation of Deferred@118935050 after retry RPC after error@923805919) -> (continuation of Deferred@1585452208 after retry RPC after error@914760472) -> (continuation of Deferred@122801500 after retry RPC after error@825314233) -> (continuation of Deferred@1867185339 after retry RPC after error@1750305961)))
06:33:52.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:52.970 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1601930811(state=PENDING, result=null, callback=(continuation of Deferred@1325498223 after retry RPC after error@338656749) -> (continuation of Deferred@1904565190 after retry RPC after error@1917604243) -> (continuation of Deferred@913482418 after retry RPC after error@1507680644) -> (continuation of Deferred@39668242 after retry RPC after error@311247619) -> (continuation of Deferred@835261650 after retry RPC after error@53117486) -> (continuation of Deferred@1808749955 after retry RPC after error@321029794) -> (continuation of Deferred@1781825424 after retry RPC after error@1029221130), errback=(continuation of Deferred@1325498223 after retry RPC after error@338656749) -> (continuation of Deferred@1904565190 after retry RPC after error@1917604243) -> (continuation of Deferred@913482418 after retry RPC after error@1507680644) -> (continuation of Deferred@39668242 after retry RPC after error@311247619) -> (continuation of Deferred@835261650 after retry RPC after error@53117486) -> (continuation of Deferred@1808749955 after retry RPC after error@321029794) -> (continuation of Deferred@1781825424 after retry RPC after error@1029221130)))
06:33:52.971 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1601930811(state=PENDING, result=null, callback=(continuation of Deferred@1325498223 after retry RPC after error@338656749) -> (continuation of Deferred@1904565190 after retry RPC after error@1917604243) -> (continuation of Deferred@913482418 after retry RPC after error@1507680644) -> (continuation of Deferred@39668242 after retry RPC after error@311247619) -> (continuation of Deferred@835261650 after retry RPC after error@53117486) -> (continuation of Deferred@1808749955 after retry RPC after error@321029794) -> (continuation of Deferred@1781825424 after retry RPC after error@1029221130), errback=(continuation of Deferred@1325498223 after retry RPC after error@338656749) -> (continuation of Deferred@1904565190 after retry RPC after error@1917604243) -> (continuation of Deferred@913482418 after retry RPC after error@1507680644) -> (continuation of Deferred@39668242 after retry RPC after error@311247619) -> (continuation of Deferred@835261650 after retry RPC after error@53117486) -> (continuation of Deferred@1808749955 after retry RPC after error@321029794) -> (continuation of Deferred@1781825424 after retry RPC after error@1029221130)))
06:33:52.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:52.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:52.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:52.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:52.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.248 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@969630283(state=PENDING, result=null, callback=(continuation of Deferred@583525182 after retry RPC after error@807736909) -> (continuation of Deferred@303186742 after retry RPC after error@1890165253) -> (continuation of Deferred@1835688253 after retry RPC after error@1969587619) -> (continuation of Deferred@1978557386 after retry RPC after error@1214930680) -> (continuation of Deferred@1983364417 after retry RPC after error@799434425) -> (continuation of Deferred@693153972 after retry RPC after error@869771000) -> (continuation of Deferred@2025969782 after retry RPC after error@1691282041), errback=(continuation of Deferred@583525182 after retry RPC after error@807736909) -> (continuation of Deferred@303186742 after retry RPC after error@1890165253) -> (continuation of Deferred@1835688253 after retry RPC after error@1969587619) -> (continuation of Deferred@1978557386 after retry RPC after error@1214930680) -> (continuation of Deferred@1983364417 after retry RPC after error@799434425) -> (continuation of Deferred@693153972 after retry RPC after error@869771000) -> (continuation of Deferred@2025969782 after retry RPC after error@1691282041)))
06:33:53.249 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:53.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.351 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.351 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.351 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.471 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@442437377(state=PENDING, result=null, callback=(continuation of Deferred@435210792 after retry RPC after error@1086070445) -> (continuation of Deferred@1858375580 after retry RPC after error@201437762) -> (continuation of Deferred@331323488 after retry RPC after error@861316556) -> (continuation of Deferred@2032428117 after retry RPC after error@2025216657) -> (continuation of Deferred@388346239 after retry RPC after error@291184274) -> (continuation of Deferred@1258800672 after retry RPC after error@28274435) -> (continuation of Deferred@1002145792 after retry RPC after error@1918746219), errback=(continuation of Deferred@435210792 after retry RPC after error@1086070445) -> (continuation of Deferred@1858375580 after retry RPC after error@201437762) -> (continuation of Deferred@331323488 after retry RPC after error@861316556) -> (continuation of Deferred@2032428117 after retry RPC after error@2025216657) -> (continuation of Deferred@388346239 after retry RPC after error@291184274) -> (continuation of Deferred@1258800672 after retry RPC after error@28274435) -> (continuation of Deferred@1002145792 after retry RPC after error@1918746219)))
06:33:53.471 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@442437377(state=PENDING, result=null, callback=(continuation of Deferred@435210792 after retry RPC after error@1086070445) -> (continuation of Deferred@1858375580 after retry RPC after error@201437762) -> (continuation of Deferred@331323488 after retry RPC after error@861316556) -> (continuation of Deferred@2032428117 after retry RPC after error@2025216657) -> (continuation of Deferred@388346239 after retry RPC after error@291184274) -> (continuation of Deferred@1258800672 after retry RPC after error@28274435) -> (continuation of Deferred@1002145792 after retry RPC after error@1918746219), errback=(continuation of Deferred@435210792 after retry RPC after error@1086070445) -> (continuation of Deferred@1858375580 after retry RPC after error@201437762) -> (continuation of Deferred@331323488 after retry RPC after error@861316556) -> (continuation of Deferred@2032428117 after retry RPC after error@2025216657) -> (continuation of Deferred@388346239 after retry RPC after error@291184274) -> (continuation of Deferred@1258800672 after retry RPC after error@28274435) -> (continuation of Deferred@1002145792 after retry RPC after error@1918746219)))
06:33:53.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.510 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.510 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.568 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.711 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1363217290(state=PENDING, result=null, callback=(continuation of Deferred@811596296 after retry RPC after error@826923054) -> (continuation of Deferred@624453407 after retry RPC after error@2096899433) -> (continuation of Deferred@867962692 after retry RPC after error@1279994293) -> (continuation of Deferred@716029884 after retry RPC after error@929501421) -> (continuation of Deferred@1685253994 after retry RPC after error@1902208208) -> (continuation of Deferred@922117407 after retry RPC after error@966388564) -> (continuation of Deferred@1248025266 after retry RPC after error@811524298), errback=(continuation of Deferred@811596296 after retry RPC after error@826923054) -> (continuation of Deferred@624453407 after retry RPC after error@2096899433) -> (continuation of Deferred@867962692 after retry RPC after error@1279994293) -> (continuation of Deferred@716029884 after retry RPC after error@929501421) -> (continuation of Deferred@1685253994 after retry RPC after error@1902208208) -> (continuation of Deferred@922117407 after retry RPC after error@966388564) -> (continuation of Deferred@1248025266 after retry RPC after error@811524298)))
06:33:53.711 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1363217290(state=PENDING, result=null, callback=(continuation of Deferred@811596296 after retry RPC after error@826923054) -> (continuation of Deferred@624453407 after retry RPC after error@2096899433) -> (continuation of Deferred@867962692 after retry RPC after error@1279994293) -> (continuation of Deferred@716029884 after retry RPC after error@929501421) -> (continuation of Deferred@1685253994 after retry RPC after error@1902208208) -> (continuation of Deferred@922117407 after retry RPC after error@966388564) -> (continuation of Deferred@1248025266 after retry RPC after error@811524298), errback=(continuation of Deferred@811596296 after retry RPC after error@826923054) -> (continuation of Deferred@624453407 after retry RPC after error@2096899433) -> (continuation of Deferred@867962692 after retry RPC after error@1279994293) -> (continuation of Deferred@716029884 after retry RPC after error@929501421) -> (continuation of Deferred@1685253994 after retry RPC after error@1902208208) -> (continuation of Deferred@922117407 after retry RPC after error@966388564) -> (continuation of Deferred@1248025266 after retry RPC after error@811524298)))
06:33:53.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.828 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.828 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1634764999(state=PENDING, result=null, callback=(continuation of Deferred@1068415813 after retry RPC after error@797176232) -> (continuation of Deferred@1918667938 after retry RPC after error@395248798) -> (continuation of Deferred@1563554233 after retry RPC after error@1838105862) -> (continuation of Deferred@2065759625 after retry RPC after error@342142343) -> (continuation of Deferred@1175075467 after retry RPC after error@1362812985) -> (continuation of Deferred@764052258 after retry RPC after error@473107329) -> (continuation of Deferred@200979694 after retry RPC after error@231188096) -> (continuation of Deferred@1957537673 after retry RPC after error@1883690398), errback=(continuation of Deferred@1068415813 after retry RPC after error@797176232) -> (continuation of Deferred@1918667938 after retry RPC after error@395248798) -> (continuation of Deferred@1563554233 after retry RPC after error@1838105862) -> (continuation of Deferred@2065759625 after retry RPC after error@342142343) -> (continuation of Deferred@1175075467 after retry RPC after error@1362812985) -> (continuation of Deferred@764052258 after retry RPC after error@473107329) -> (continuation of Deferred@200979694 after retry RPC after error@231188096) -> (continuation of Deferred@1957537673 after retry RPC after error@1883690398)))
06:33:53.951 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1634764999(state=PENDING, result=null, callback=(continuation of Deferred@1068415813 after retry RPC after error@797176232) -> (continuation of Deferred@1918667938 after retry RPC after error@395248798) -> (continuation of Deferred@1563554233 after retry RPC after error@1838105862) -> (continuation of Deferred@2065759625 after retry RPC after error@342142343) -> (continuation of Deferred@1175075467 after retry RPC after error@1362812985) -> (continuation of Deferred@764052258 after retry RPC after error@473107329) -> (continuation of Deferred@200979694 after retry RPC after error@231188096) -> (continuation of Deferred@1957537673 after retry RPC after error@1883690398), errback=(continuation of Deferred@1068415813 after retry RPC after error@797176232) -> (continuation of Deferred@1918667938 after retry RPC after error@395248798) -> (continuation of Deferred@1563554233 after retry RPC after error@1838105862) -> (continuation of Deferred@2065759625 after retry RPC after error@342142343) -> (continuation of Deferred@1175075467 after retry RPC after error@1362812985) -> (continuation of Deferred@764052258 after retry RPC after error@473107329) -> (continuation of Deferred@200979694 after retry RPC after error@231188096) -> (continuation of Deferred@1957537673 after retry RPC after error@1883690398)))
06:33:53.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:53.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:53.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:53.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:53.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:53.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.111 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1794247575(state=PENDING, result=null, callback=(continuation of Deferred@424423592 after retry RPC after error@1215567441) -> (continuation of Deferred@1593712931 after retry RPC after error@606742488) -> (continuation of Deferred@1545308805 after retry RPC after error@875944393) -> (continuation of Deferred@468517205 after retry RPC after error@1846337815) -> (continuation of Deferred@1858273777 after retry RPC after error@1948415601) -> (continuation of Deferred@120989193 after retry RPC after error@238779297), errback=(continuation of Deferred@424423592 after retry RPC after error@1215567441) -> (continuation of Deferred@1593712931 after retry RPC after error@606742488) -> (continuation of Deferred@1545308805 after retry RPC after error@875944393) -> (continuation of Deferred@468517205 after retry RPC after error@1846337815) -> (continuation of Deferred@1858273777 after retry RPC after error@1948415601) -> (continuation of Deferred@120989193 after retry RPC after error@238779297)))
06:33:54.111 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1794247575(state=PENDING, result=null, callback=(continuation of Deferred@424423592 after retry RPC after error@1215567441) -> (continuation of Deferred@1593712931 after retry RPC after error@606742488) -> (continuation of Deferred@1545308805 after retry RPC after error@875944393) -> (continuation of Deferred@468517205 after retry RPC after error@1846337815) -> (continuation of Deferred@1858273777 after retry RPC after error@1948415601) -> (continuation of Deferred@120989193 after retry RPC after error@238779297), errback=(continuation of Deferred@424423592 after retry RPC after error@1215567441) -> (continuation of Deferred@1593712931 after retry RPC after error@606742488) -> (continuation of Deferred@1545308805 after retry RPC after error@875944393) -> (continuation of Deferred@468517205 after retry RPC after error@1846337815) -> (continuation of Deferred@1858273777 after retry RPC after error@1948415601) -> (continuation of Deferred@120989193 after retry RPC after error@238779297)))
06:33:54.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.168 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.310 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@170739509(state=PENDING, result=null, callback=(continuation of Deferred@1323067944 after retry RPC after error@347319669) -> (continuation of Deferred@2128958046 after retry RPC after error@512861784) -> (continuation of Deferred@858849928 after retry RPC after error@979425694) -> (continuation of Deferred@1849891630 after retry RPC after error@2104738776) -> (continuation of Deferred@1189981116 after retry RPC after error@2063414998) -> (continuation of Deferred@210880716 after retry RPC after error@1864349573), errback=(continuation of Deferred@1323067944 after retry RPC after error@347319669) -> (continuation of Deferred@2128958046 after retry RPC after error@512861784) -> (continuation of Deferred@858849928 after retry RPC after error@979425694) -> (continuation of Deferred@1849891630 after retry RPC after error@2104738776) -> (continuation of Deferred@1189981116 after retry RPC after error@2063414998) -> (continuation of Deferred@210880716 after retry RPC after error@1864349573)))
06:33:54.311 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@170739509(state=PENDING, result=null, callback=(continuation of Deferred@1323067944 after retry RPC after error@347319669) -> (continuation of Deferred@2128958046 after retry RPC after error@512861784) -> (continuation of Deferred@858849928 after retry RPC after error@979425694) -> (continuation of Deferred@1849891630 after retry RPC after error@2104738776) -> (continuation of Deferred@1189981116 after retry RPC after error@2063414998) -> (continuation of Deferred@210880716 after retry RPC after error@1864349573), errback=(continuation of Deferred@1323067944 after retry RPC after error@347319669) -> (continuation of Deferred@2128958046 after retry RPC after error@512861784) -> (continuation of Deferred@858849928 after retry RPC after error@979425694) -> (continuation of Deferred@1849891630 after retry RPC after error@2104738776) -> (continuation of Deferred@1189981116 after retry RPC after error@2063414998) -> (continuation of Deferred@210880716 after retry RPC after error@1864349573)))
06:33:54.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.428 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.490 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.490 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.490 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.490 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.490 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.570 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@239657538(state=PENDING, result=null, callback=(continuation of Deferred@132583421 after retry RPC after error@205255390) -> (continuation of Deferred@1016081754 after retry RPC after error@1963168376) -> (continuation of Deferred@1243931586 after retry RPC after error@425350490) -> (continuation of Deferred@614289878 after retry RPC after error@1298437999) -> (continuation of Deferred@548624370 after retry RPC after error@1804088891) -> (continuation of Deferred@626379133 after retry RPC after error@1290005847) -> (continuation of Deferred@1993687038 after retry RPC after error@1556543217), errback=(continuation of Deferred@132583421 after retry RPC after error@205255390) -> (continuation of Deferred@1016081754 after retry RPC after error@1963168376) -> (continuation of Deferred@1243931586 after retry RPC after error@425350490) -> (continuation of Deferred@614289878 after retry RPC after error@1298437999) -> (continuation of Deferred@548624370 after retry RPC after error@1804088891) -> (continuation of Deferred@626379133 after retry RPC after error@1290005847) -> (continuation of Deferred@1993687038 after retry RPC after error@1556543217)))
06:33:54.571 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@239657538(state=PENDING, result=null, callback=(continuation of Deferred@132583421 after retry RPC after error@205255390) -> (continuation of Deferred@1016081754 after retry RPC after error@1963168376) -> (continuation of Deferred@1243931586 after retry RPC after error@425350490) -> (continuation of Deferred@614289878 after retry RPC after error@1298437999) -> (continuation of Deferred@548624370 after retry RPC after error@1804088891) -> (continuation of Deferred@626379133 after retry RPC after error@1290005847) -> (continuation of Deferred@1993687038 after retry RPC after error@1556543217), errback=(continuation of Deferred@132583421 after retry RPC after error@205255390) -> (continuation of Deferred@1016081754 after retry RPC after error@1963168376) -> (continuation of Deferred@1243931586 after retry RPC after error@425350490) -> (continuation of Deferred@614289878 after retry RPC after error@1298437999) -> (continuation of Deferred@548624370 after retry RPC after error@1804088891) -> (continuation of Deferred@626379133 after retry RPC after error@1290005847) -> (continuation of Deferred@1993687038 after retry RPC after error@1556543217)))
06:33:54.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.751 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.751 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@273684023(state=PENDING, result=null, callback=(continuation of Deferred@1723482721 after retry RPC after error@1791813798) -> (continuation of Deferred@644632308 after retry RPC after error@916264352) -> (continuation of Deferred@33611864 after retry RPC after error@260350310) -> (continuation of Deferred@1264272780 after retry RPC after error@355821906) -> (continuation of Deferred@280119692 after retry RPC after error@982992237) -> (continuation of Deferred@728830102 after retry RPC after error@363889491), errback=(continuation of Deferred@1723482721 after retry RPC after error@1791813798) -> (continuation of Deferred@644632308 after retry RPC after error@916264352) -> (continuation of Deferred@33611864 after retry RPC after error@260350310) -> (continuation of Deferred@1264272780 after retry RPC after error@355821906) -> (continuation of Deferred@280119692 after retry RPC after error@982992237) -> (continuation of Deferred@728830102 after retry RPC after error@363889491)))
06:33:54.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@273684023(state=PENDING, result=null, callback=(continuation of Deferred@1723482721 after retry RPC after error@1791813798) -> (continuation of Deferred@644632308 after retry RPC after error@916264352) -> (continuation of Deferred@33611864 after retry RPC after error@260350310) -> (continuation of Deferred@1264272780 after retry RPC after error@355821906) -> (continuation of Deferred@280119692 after retry RPC after error@982992237) -> (continuation of Deferred@728830102 after retry RPC after error@363889491), errback=(continuation of Deferred@1723482721 after retry RPC after error@1791813798) -> (continuation of Deferred@644632308 after retry RPC after error@916264352) -> (continuation of Deferred@33611864 after retry RPC after error@260350310) -> (continuation of Deferred@1264272780 after retry RPC after error@355821906) -> (continuation of Deferred@280119692 after retry RPC after error@982992237) -> (continuation of Deferred@728830102 after retry RPC after error@363889491)))
06:33:54.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.930 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.930 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:54.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:54.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:54.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:54.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:54.990 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@628109082(state=PENDING, result=null, callback=(continuation of Deferred@761825680 after retry RPC after error@1787120037) -> (continuation of Deferred@413770182 after retry RPC after error@1006982454) -> (continuation of Deferred@1439199023 after retry RPC after error@364956754) -> (continuation of Deferred@1077078205 after retry RPC after error@1443412328) -> (continuation of Deferred@1085216145 after retry RPC after error@1402921866) -> (continuation of Deferred@79597124 after retry RPC after error@393126978) -> (continuation of Deferred@794110115 after retry RPC after error@200477808), errback=(continuation of Deferred@761825680 after retry RPC after error@1787120037) -> (continuation of Deferred@413770182 after retry RPC after error@1006982454) -> (continuation of Deferred@1439199023 after retry RPC after error@364956754) -> (continuation of Deferred@1077078205 after retry RPC after error@1443412328) -> (continuation of Deferred@1085216145 after retry RPC after error@1402921866) -> (continuation of Deferred@79597124 after retry RPC after error@393126978) -> (continuation of Deferred@794110115 after retry RPC after error@200477808)))
06:33:54.991 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@628109082(state=PENDING, result=null, callback=(continuation of Deferred@761825680 after retry RPC after error@1787120037) -> (continuation of Deferred@413770182 after retry RPC after error@1006982454) -> (continuation of Deferred@1439199023 after retry RPC after error@364956754) -> (continuation of Deferred@1077078205 after retry RPC after error@1443412328) -> (continuation of Deferred@1085216145 after retry RPC after error@1402921866) -> (continuation of Deferred@79597124 after retry RPC after error@393126978) -> (continuation of Deferred@794110115 after retry RPC after error@200477808), errback=(continuation of Deferred@761825680 after retry RPC after error@1787120037) -> (continuation of Deferred@413770182 after retry RPC after error@1006982454) -> (continuation of Deferred@1439199023 after retry RPC after error@364956754) -> (continuation of Deferred@1077078205 after retry RPC after error@1443412328) -> (continuation of Deferred@1085216145 after retry RPC after error@1402921866) -> (continuation of Deferred@79597124 after retry RPC after error@393126978) -> (continuation of Deferred@794110115 after retry RPC after error@200477808)))
06:33:55.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.024 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.024696 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 86) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:55.027 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.027119 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 86) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:33:55.028 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.028532 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:33:55.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.090 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.090 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:55.092317 25597 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:33:55.092 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:55.092540 25597 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:33:55.093 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:55.093286 25597 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:33:55.095 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.096 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.095861 25333 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111) [suppressed 5 similar messages]
06:33:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.097851 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:33:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:33:55.098065 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:33:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:55.098220 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:33:55.098 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:55.098572 25597 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:33:55.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [87ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [87ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1308262030(state=PENDING, result=null, callback=(continuation of Deferred@273330155 after retry RPC after error@690765229) -> (continuation of Deferred@1865277019 after retry RPC after error@1908265790) -> (continuation of Deferred@885853973 after retry RPC after error@2093256052) -> (continuation of Deferred@2098879409 after retry RPC after error@1646351119) -> (continuation of Deferred@1044925945 after retry RPC after error@1723848809) -> (continuation of Deferred@1238880476 after retry RPC after error@554060738), errback=(continuation of Deferred@273330155 after retry RPC after error@690765229) -> (continuation of Deferred@1865277019 after retry RPC after error@1908265790) -> (continuation of Deferred@885853973 after retry RPC after error@2093256052) -> (continuation of Deferred@2098879409 after retry RPC after error@1646351119) -> (continuation of Deferred@1044925945 after retry RPC after error@1723848809) -> (continuation of Deferred@1238880476 after retry RPC after error@554060738)))
06:33:55.171 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [87ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [87ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1308262030(state=PENDING, result=null, callback=(continuation of Deferred@273330155 after retry RPC after error@690765229) -> (continuation of Deferred@1865277019 after retry RPC after error@1908265790) -> (continuation of Deferred@885853973 after retry RPC after error@2093256052) -> (continuation of Deferred@2098879409 after retry RPC after error@1646351119) -> (continuation of Deferred@1044925945 after retry RPC after error@1723848809) -> (continuation of Deferred@1238880476 after retry RPC after error@554060738), errback=(continuation of Deferred@273330155 after retry RPC after error@690765229) -> (continuation of Deferred@1865277019 after retry RPC after error@1908265790) -> (continuation of Deferred@885853973 after retry RPC after error@2093256052) -> (continuation of Deferred@2098879409 after retry RPC after error@1646351119) -> (continuation of Deferred@1044925945 after retry RPC after error@1723848809) -> (continuation of Deferred@1238880476 after retry RPC after error@554060738)))
06:33:55.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.188 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.211 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.211 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.348 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.349 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.350 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@953054062(state=PENDING, result=null, callback=(continuation of Deferred@625369073 after retry RPC after error@1190771709) -> (continuation of Deferred@1043212387 after retry RPC after error@1966111179) -> (continuation of Deferred@1654528331 after retry RPC after error@1764974774) -> (continuation of Deferred@638656707 after retry RPC after error@1343876075) -> (continuation of Deferred@1121973474 after retry RPC after error@815467231) -> (continuation of Deferred@530789175 after retry RPC after error@8050075) -> (continuation of Deferred@1027557778 after retry RPC after error@1436184856), errback=(continuation of Deferred@625369073 after retry RPC after error@1190771709) -> (continuation of Deferred@1043212387 after retry RPC after error@1966111179) -> (continuation of Deferred@1654528331 after retry RPC after error@1764974774) -> (continuation of Deferred@638656707 after retry RPC after error@1343876075) -> (continuation of Deferred@1121973474 after retry RPC after error@815467231) -> (continuation of Deferred@530789175 after retry RPC after error@8050075) -> (continuation of Deferred@1027557778 after retry RPC after error@1436184856)))
06:33:55.350 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@953054062(state=PENDING, result=null, callback=(continuation of Deferred@625369073 after retry RPC after error@1190771709) -> (continuation of Deferred@1043212387 after retry RPC after error@1966111179) -> (continuation of Deferred@1654528331 after retry RPC after error@1764974774) -> (continuation of Deferred@638656707 after retry RPC after error@1343876075) -> (continuation of Deferred@1121973474 after retry RPC after error@815467231) -> (continuation of Deferred@530789175 after retry RPC after error@8050075) -> (continuation of Deferred@1027557778 after retry RPC after error@1436184856), errback=(continuation of Deferred@625369073 after retry RPC after error@1190771709) -> (continuation of Deferred@1043212387 after retry RPC after error@1966111179) -> (continuation of Deferred@1654528331 after retry RPC after error@1764974774) -> (continuation of Deferred@638656707 after retry RPC after error@1343876075) -> (continuation of Deferred@1121973474 after retry RPC after error@815467231) -> (continuation of Deferred@530789175 after retry RPC after error@8050075) -> (continuation of Deferred@1027557778 after retry RPC after error@1436184856)))
06:33:55.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.370 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@678604143(state=PENDING, result=null, callback=(continuation of Deferred@1980802150 after retry RPC after error@1836966595) -> (continuation of Deferred@1869045809 after retry RPC after error@1388552551) -> (continuation of Deferred@562240717 after retry RPC after error@594188740) -> (continuation of Deferred@2002260848 after retry RPC after error@945769573) -> (continuation of Deferred@25381591 after retry RPC after error@1984976794) -> (continuation of Deferred@1092880819 after retry RPC after error@56248726) -> (continuation of Deferred@412384984 after retry RPC after error@1821116119), errback=(continuation of Deferred@1980802150 after retry RPC after error@1836966595) -> (continuation of Deferred@1869045809 after retry RPC after error@1388552551) -> (continuation of Deferred@562240717 after retry RPC after error@594188740) -> (continuation of Deferred@2002260848 after retry RPC after error@945769573) -> (continuation of Deferred@25381591 after retry RPC after error@1984976794) -> (continuation of Deferred@1092880819 after retry RPC after error@56248726) -> (continuation of Deferred@412384984 after retry RPC after error@1821116119)))
06:33:55.611 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@678604143(state=PENDING, result=null, callback=(continuation of Deferred@1980802150 after retry RPC after error@1836966595) -> (continuation of Deferred@1869045809 after retry RPC after error@1388552551) -> (continuation of Deferred@562240717 after retry RPC after error@594188740) -> (continuation of Deferred@2002260848 after retry RPC after error@945769573) -> (continuation of Deferred@25381591 after retry RPC after error@1984976794) -> (continuation of Deferred@1092880819 after retry RPC after error@56248726) -> (continuation of Deferred@412384984 after retry RPC after error@1821116119), errback=(continuation of Deferred@1980802150 after retry RPC after error@1836966595) -> (continuation of Deferred@1869045809 after retry RPC after error@1388552551) -> (continuation of Deferred@562240717 after retry RPC after error@594188740) -> (continuation of Deferred@2002260848 after retry RPC after error@945769573) -> (continuation of Deferred@25381591 after retry RPC after error@1984976794) -> (continuation of Deferred@1092880819 after retry RPC after error@56248726) -> (continuation of Deferred@412384984 after retry RPC after error@1821116119)))
06:33:55.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.708 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.730 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.730 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.731 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@391299980(state=PENDING, result=null, callback=(continuation of Deferred@1941777574 after retry RPC after error@653448562) -> (continuation of Deferred@1548639380 after retry RPC after error@1560394290) -> (continuation of Deferred@1203991848 after retry RPC after error@1999266279) -> (continuation of Deferred@1504586090 after retry RPC after error@154060871) -> (continuation of Deferred@1427670655 after retry RPC after error@1905909858) -> (continuation of Deferred@1507095106 after retry RPC after error@642250191) -> (continuation of Deferred@930980870 after retry RPC after error@156878971), errback=(continuation of Deferred@1941777574 after retry RPC after error@653448562) -> (continuation of Deferred@1548639380 after retry RPC after error@1560394290) -> (continuation of Deferred@1203991848 after retry RPC after error@1999266279) -> (continuation of Deferred@1504586090 after retry RPC after error@154060871) -> (continuation of Deferred@1427670655 after retry RPC after error@1905909858) -> (continuation of Deferred@1507095106 after retry RPC after error@642250191) -> (continuation of Deferred@930980870 after retry RPC after error@156878971)))
06:33:55.812 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=183), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@391299980(state=PENDING, result=null, callback=(continuation of Deferred@1941777574 after retry RPC after error@653448562) -> (continuation of Deferred@1548639380 after retry RPC after error@1560394290) -> (continuation of Deferred@1203991848 after retry RPC after error@1999266279) -> (continuation of Deferred@1504586090 after retry RPC after error@154060871) -> (continuation of Deferred@1427670655 after retry RPC after error@1905909858) -> (continuation of Deferred@1507095106 after retry RPC after error@642250191) -> (continuation of Deferred@930980870 after retry RPC after error@156878971), errback=(continuation of Deferred@1941777574 after retry RPC after error@653448562) -> (continuation of Deferred@1548639380 after retry RPC after error@1560394290) -> (continuation of Deferred@1203991848 after retry RPC after error@1999266279) -> (continuation of Deferred@1504586090 after retry RPC after error@154060871) -> (continuation of Deferred@1427670655 after retry RPC after error@1905909858) -> (continuation of Deferred@1507095106 after retry RPC after error@642250191) -> (continuation of Deferred@930980870 after retry RPC after error@156878971)))
06:33:55.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:55.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:55.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:55.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:55.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:55.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@143741360(state=PENDING, result=null, callback=(continuation of Deferred@1060549159 after retry RPC after error@935072353) -> (continuation of Deferred@1813127072 after retry RPC after error@1854721859) -> (continuation of Deferred@1679945535 after retry RPC after error@869914304) -> (continuation of Deferred@1330262020 after retry RPC after error@1202980985) -> (continuation of Deferred@747069351 after retry RPC after error@1368781601) -> (continuation of Deferred@1338061575 after retry RPC after error@676400454), errback=(continuation of Deferred@1060549159 after retry RPC after error@935072353) -> (continuation of Deferred@1813127072 after retry RPC after error@1854721859) -> (continuation of Deferred@1679945535 after retry RPC after error@869914304) -> (continuation of Deferred@1330262020 after retry RPC after error@1202980985) -> (continuation of Deferred@747069351 after retry RPC after error@1368781601) -> (continuation of Deferred@1338061575 after retry RPC after error@676400454)))
06:33:56.031 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@143741360(state=PENDING, result=null, callback=(continuation of Deferred@1060549159 after retry RPC after error@935072353) -> (continuation of Deferred@1813127072 after retry RPC after error@1854721859) -> (continuation of Deferred@1679945535 after retry RPC after error@869914304) -> (continuation of Deferred@1330262020 after retry RPC after error@1202980985) -> (continuation of Deferred@747069351 after retry RPC after error@1368781601) -> (continuation of Deferred@1338061575 after retry RPC after error@676400454), errback=(continuation of Deferred@1060549159 after retry RPC after error@935072353) -> (continuation of Deferred@1813127072 after retry RPC after error@1854721859) -> (continuation of Deferred@1679945535 after retry RPC after error@869914304) -> (continuation of Deferred@1330262020 after retry RPC after error@1202980985) -> (continuation of Deferred@747069351 after retry RPC after error@1368781601) -> (continuation of Deferred@1338061575 after retry RPC after error@676400454)))
06:33:56.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.292 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@749297450(state=PENDING, result=null, callback=(continuation of Deferred@242724434 after retry RPC after error@1360587133) -> (continuation of Deferred@417078603 after retry RPC after error@2129285493) -> (continuation of Deferred@1306293700 after retry RPC after error@1528380931) -> (continuation of Deferred@592922222 after retry RPC after error@261806728) -> (continuation of Deferred@566684906 after retry RPC after error@868038647) -> (continuation of Deferred@708862936 after retry RPC after error@1935123074) -> (continuation of Deferred@1700831678 after retry RPC after error@1069031530) -> (continuation of Deferred@374328038 after retry RPC after error@1480735007), errback=(continuation of Deferred@242724434 after retry RPC after error@1360587133) -> (continuation of Deferred@417078603 after retry RPC after error@2129285493) -> (continuation of Deferred@1306293700 after retry RPC after error@1528380931) -> (continuation of Deferred@592922222 after retry RPC after error@261806728) -> (continuation of Deferred@566684906 after retry RPC after error@868038647) -> (continuation of Deferred@708862936 after retry RPC after error@1935123074) -> (continuation of Deferred@1700831678 after retry RPC after error@1069031530) -> (continuation of Deferred@374328038 after retry RPC after error@1480735007)))
06:33:56.293 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@749297450(state=PENDING, result=null, callback=(continuation of Deferred@242724434 after retry RPC after error@1360587133) -> (continuation of Deferred@417078603 after retry RPC after error@2129285493) -> (continuation of Deferred@1306293700 after retry RPC after error@1528380931) -> (continuation of Deferred@592922222 after retry RPC after error@261806728) -> (continuation of Deferred@566684906 after retry RPC after error@868038647) -> (continuation of Deferred@708862936 after retry RPC after error@1935123074) -> (continuation of Deferred@1700831678 after retry RPC after error@1069031530) -> (continuation of Deferred@374328038 after retry RPC after error@1480735007), errback=(continuation of Deferred@242724434 after retry RPC after error@1360587133) -> (continuation of Deferred@417078603 after retry RPC after error@2129285493) -> (continuation of Deferred@1306293700 after retry RPC after error@1528380931) -> (continuation of Deferred@592922222 after retry RPC after error@261806728) -> (continuation of Deferred@566684906 after retry RPC after error@868038647) -> (continuation of Deferred@708862936 after retry RPC after error@1935123074) -> (continuation of Deferred@1700831678 after retry RPC after error@1069031530) -> (continuation of Deferred@374328038 after retry RPC after error@1480735007)))
06:33:56.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.348 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.348 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.349 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.388 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.530 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.530 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.531 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@351302389(state=PENDING, result=null, callback=(continuation of Deferred@1868192526 after retry RPC after error@692718639) -> (continuation of Deferred@1781102144 after retry RPC after error@2111424042) -> (continuation of Deferred@1088843956 after retry RPC after error@968108019) -> (continuation of Deferred@403623411 after retry RPC after error@735424590) -> (continuation of Deferred@1489307216 after retry RPC after error@1087781124) -> (continuation of Deferred@785978808 after retry RPC after error@671651821) -> (continuation of Deferred@1485824826 after retry RPC after error@702607158), errback=(continuation of Deferred@1868192526 after retry RPC after error@692718639) -> (continuation of Deferred@1781102144 after retry RPC after error@2111424042) -> (continuation of Deferred@1088843956 after retry RPC after error@968108019) -> (continuation of Deferred@403623411 after retry RPC after error@735424590) -> (continuation of Deferred@1489307216 after retry RPC after error@1087781124) -> (continuation of Deferred@785978808 after retry RPC after error@671651821) -> (continuation of Deferred@1485824826 after retry RPC after error@702607158)))
06:33:56.531 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@351302389(state=PENDING, result=null, callback=(continuation of Deferred@1868192526 after retry RPC after error@692718639) -> (continuation of Deferred@1781102144 after retry RPC after error@2111424042) -> (continuation of Deferred@1088843956 after retry RPC after error@968108019) -> (continuation of Deferred@403623411 after retry RPC after error@735424590) -> (continuation of Deferred@1489307216 after retry RPC after error@1087781124) -> (continuation of Deferred@785978808 after retry RPC after error@671651821) -> (continuation of Deferred@1485824826 after retry RPC after error@702607158), errback=(continuation of Deferred@1868192526 after retry RPC after error@692718639) -> (continuation of Deferred@1781102144 after retry RPC after error@2111424042) -> (continuation of Deferred@1088843956 after retry RPC after error@968108019) -> (continuation of Deferred@403623411 after retry RPC after error@735424590) -> (continuation of Deferred@1489307216 after retry RPC after error@1087781124) -> (continuation of Deferred@785978808 after retry RPC after error@671651821) -> (continuation of Deferred@1485824826 after retry RPC after error@702607158)))
06:33:56.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.669 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2127308178(state=PENDING, result=null, callback=(continuation of Deferred@1172458553 after retry RPC after error@1602527714) -> (continuation of Deferred@1799679152 after retry RPC after error@1950065919) -> (continuation of Deferred@602910149 after retry RPC after error@1495176171) -> (continuation of Deferred@368301332 after retry RPC after error@1011255049) -> (continuation of Deferred@1654228161 after retry RPC after error@1337311922) -> (continuation of Deferred@1155921189 after retry RPC after error@1284725887) -> (continuation of Deferred@1652717216 after retry RPC after error@54757392), errback=(continuation of Deferred@1172458553 after retry RPC after error@1602527714) -> (continuation of Deferred@1799679152 after retry RPC after error@1950065919) -> (continuation of Deferred@602910149 after retry RPC after error@1495176171) -> (continuation of Deferred@368301332 after retry RPC after error@1011255049) -> (continuation of Deferred@1654228161 after retry RPC after error@1337311922) -> (continuation of Deferred@1155921189 after retry RPC after error@1284725887) -> (continuation of Deferred@1652717216 after retry RPC after error@54757392)))
06:33:56.692 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2127308178(state=PENDING, result=null, callback=(continuation of Deferred@1172458553 after retry RPC after error@1602527714) -> (continuation of Deferred@1799679152 after retry RPC after error@1950065919) -> (continuation of Deferred@602910149 after retry RPC after error@1495176171) -> (continuation of Deferred@368301332 after retry RPC after error@1011255049) -> (continuation of Deferred@1654228161 after retry RPC after error@1337311922) -> (continuation of Deferred@1155921189 after retry RPC after error@1284725887) -> (continuation of Deferred@1652717216 after retry RPC after error@54757392), errback=(continuation of Deferred@1172458553 after retry RPC after error@1602527714) -> (continuation of Deferred@1799679152 after retry RPC after error@1950065919) -> (continuation of Deferred@602910149 after retry RPC after error@1495176171) -> (continuation of Deferred@368301332 after retry RPC after error@1011255049) -> (continuation of Deferred@1654228161 after retry RPC after error@1337311922) -> (continuation of Deferred@1155921189 after retry RPC after error@1284725887) -> (continuation of Deferred@1652717216 after retry RPC after error@54757392)))
06:33:56.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.788 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.911 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@114305680(state=PENDING, result=null, callback=(continuation of Deferred@1600446150 after retry RPC after error@631925538) -> (continuation of Deferred@1108610591 after retry RPC after error@1159149671) -> (continuation of Deferred@1014779117 after retry RPC after error@1257886523) -> (continuation of Deferred@687822989 after retry RPC after error@1423415797) -> (continuation of Deferred@177384614 after retry RPC after error@251053764) -> (continuation of Deferred@1553976487 after retry RPC after error@879750896) -> (continuation of Deferred@721916743 after retry RPC after error@327135237), errback=(continuation of Deferred@1600446150 after retry RPC after error@631925538) -> (continuation of Deferred@1108610591 after retry RPC after error@1159149671) -> (continuation of Deferred@1014779117 after retry RPC after error@1257886523) -> (continuation of Deferred@687822989 after retry RPC after error@1423415797) -> (continuation of Deferred@177384614 after retry RPC after error@251053764) -> (continuation of Deferred@1553976487 after retry RPC after error@879750896) -> (continuation of Deferred@721916743 after retry RPC after error@327135237)))
06:33:56.911 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@114305680(state=PENDING, result=null, callback=(continuation of Deferred@1600446150 after retry RPC after error@631925538) -> (continuation of Deferred@1108610591 after retry RPC after error@1159149671) -> (continuation of Deferred@1014779117 after retry RPC after error@1257886523) -> (continuation of Deferred@687822989 after retry RPC after error@1423415797) -> (continuation of Deferred@177384614 after retry RPC after error@251053764) -> (continuation of Deferred@1553976487 after retry RPC after error@879750896) -> (continuation of Deferred@721916743 after retry RPC after error@327135237), errback=(continuation of Deferred@1600446150 after retry RPC after error@631925538) -> (continuation of Deferred@1108610591 after retry RPC after error@1159149671) -> (continuation of Deferred@1014779117 after retry RPC after error@1257886523) -> (continuation of Deferred@687822989 after retry RPC after error@1423415797) -> (continuation of Deferred@177384614 after retry RPC after error@251053764) -> (continuation of Deferred@1553976487 after retry RPC after error@879750896) -> (continuation of Deferred@721916743 after retry RPC after error@327135237)))
06:33:56.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:56.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:56.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:56.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:56.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:56.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.131 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@91291374(state=PENDING, result=null, callback=(continuation of Deferred@199740134 after retry RPC after error@2087072440) -> (continuation of Deferred@1210635126 after retry RPC after error@1252901651) -> (continuation of Deferred@2141753434 after retry RPC after error@840343269) -> (continuation of Deferred@1009469618 after retry RPC after error@406499711) -> (continuation of Deferred@590076002 after retry RPC after error@625773396) -> (continuation of Deferred@1136929175 after retry RPC after error@246995417) -> (continuation of Deferred@1932388516 after retry RPC after error@855296958), errback=(continuation of Deferred@199740134 after retry RPC after error@2087072440) -> (continuation of Deferred@1210635126 after retry RPC after error@1252901651) -> (continuation of Deferred@2141753434 after retry RPC after error@840343269) -> (continuation of Deferred@1009469618 after retry RPC after error@406499711) -> (continuation of Deferred@590076002 after retry RPC after error@625773396) -> (continuation of Deferred@1136929175 after retry RPC after error@246995417) -> (continuation of Deferred@1932388516 after retry RPC after error@855296958)))
06:33:57.131 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@91291374(state=PENDING, result=null, callback=(continuation of Deferred@199740134 after retry RPC after error@2087072440) -> (continuation of Deferred@1210635126 after retry RPC after error@1252901651) -> (continuation of Deferred@2141753434 after retry RPC after error@840343269) -> (continuation of Deferred@1009469618 after retry RPC after error@406499711) -> (continuation of Deferred@590076002 after retry RPC after error@625773396) -> (continuation of Deferred@1136929175 after retry RPC after error@246995417) -> (continuation of Deferred@1932388516 after retry RPC after error@855296958), errback=(continuation of Deferred@199740134 after retry RPC after error@2087072440) -> (continuation of Deferred@1210635126 after retry RPC after error@1252901651) -> (continuation of Deferred@2141753434 after retry RPC after error@840343269) -> (continuation of Deferred@1009469618 after retry RPC after error@406499711) -> (continuation of Deferred@590076002 after retry RPC after error@625773396) -> (continuation of Deferred@1136929175 after retry RPC after error@246995417) -> (continuation of Deferred@1932388516 after retry RPC after error@855296958)))
06:33:57.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.210 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.210 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.210 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:57.318629 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:33:57.318 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:33:57.318817 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:33:57.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@632106051(state=PENDING, result=null, callback=(continuation of Deferred@485915780 after retry RPC after error@595128024) -> (continuation of Deferred@42240713 after retry RPC after error@1317499374) -> (continuation of Deferred@1631200998 after retry RPC after error@89131575) -> (continuation of Deferred@597957270 after retry RPC after error@1448679181) -> (continuation of Deferred@1726223926 after retry RPC after error@1667279215) -> (continuation of Deferred@1611886399 after retry RPC after error@2129456463) -> (continuation of Deferred@956236034 after retry RPC after error@1580395683) -> (continuation of Deferred@623057613 after retry RPC after error@1010568627), errback=(continuation of Deferred@485915780 after retry RPC after error@595128024) -> (continuation of Deferred@42240713 after retry RPC after error@1317499374) -> (continuation of Deferred@1631200998 after retry RPC after error@89131575) -> (continuation of Deferred@597957270 after retry RPC after error@1448679181) -> (continuation of Deferred@1726223926 after retry RPC after error@1667279215) -> (continuation of Deferred@1611886399 after retry RPC after error@2129456463) -> (continuation of Deferred@956236034 after retry RPC after error@1580395683) -> (continuation of Deferred@623057613 after retry RPC after error@1010568627)))
06:33:57.351 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@632106051(state=PENDING, result=null, callback=(continuation of Deferred@485915780 after retry RPC after error@595128024) -> (continuation of Deferred@42240713 after retry RPC after error@1317499374) -> (continuation of Deferred@1631200998 after retry RPC after error@89131575) -> (continuation of Deferred@597957270 after retry RPC after error@1448679181) -> (continuation of Deferred@1726223926 after retry RPC after error@1667279215) -> (continuation of Deferred@1611886399 after retry RPC after error@2129456463) -> (continuation of Deferred@956236034 after retry RPC after error@1580395683) -> (continuation of Deferred@623057613 after retry RPC after error@1010568627), errback=(continuation of Deferred@485915780 after retry RPC after error@595128024) -> (continuation of Deferred@42240713 after retry RPC after error@1317499374) -> (continuation of Deferred@1631200998 after retry RPC after error@89131575) -> (continuation of Deferred@597957270 after retry RPC after error@1448679181) -> (continuation of Deferred@1726223926 after retry RPC after error@1667279215) -> (continuation of Deferred@1611886399 after retry RPC after error@2129456463) -> (continuation of Deferred@956236034 after retry RPC after error@1580395683) -> (continuation of Deferred@623057613 after retry RPC after error@1010568627)))
06:33:57.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.430 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.451 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.452 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@868135871(state=PENDING, result=null, callback=(continuation of Deferred@695816774 after retry RPC after error@191328401) -> (continuation of Deferred@485056375 after retry RPC after error@1519137822) -> (continuation of Deferred@956610155 after retry RPC after error@156740130) -> (continuation of Deferred@1595100824 after retry RPC after error@1678032288) -> (continuation of Deferred@2024798373 after retry RPC after error@1285380904) -> (continuation of Deferred@163556269 after retry RPC after error@1217916551) -> (continuation of Deferred@1333419488 after retry RPC after error@1763187516), errback=(continuation of Deferred@695816774 after retry RPC after error@191328401) -> (continuation of Deferred@485056375 after retry RPC after error@1519137822) -> (continuation of Deferred@956610155 after retry RPC after error@156740130) -> (continuation of Deferred@1595100824 after retry RPC after error@1678032288) -> (continuation of Deferred@2024798373 after retry RPC after error@1285380904) -> (continuation of Deferred@163556269 after retry RPC after error@1217916551) -> (continuation of Deferred@1333419488 after retry RPC after error@1763187516)))
06:33:57.571 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@868135871(state=PENDING, result=null, callback=(continuation of Deferred@695816774 after retry RPC after error@191328401) -> (continuation of Deferred@485056375 after retry RPC after error@1519137822) -> (continuation of Deferred@956610155 after retry RPC after error@156740130) -> (continuation of Deferred@1595100824 after retry RPC after error@1678032288) -> (continuation of Deferred@2024798373 after retry RPC after error@1285380904) -> (continuation of Deferred@163556269 after retry RPC after error@1217916551) -> (continuation of Deferred@1333419488 after retry RPC after error@1763187516), errback=(continuation of Deferred@695816774 after retry RPC after error@191328401) -> (continuation of Deferred@485056375 after retry RPC after error@1519137822) -> (continuation of Deferred@956610155 after retry RPC after error@156740130) -> (continuation of Deferred@1595100824 after retry RPC after error@1678032288) -> (continuation of Deferred@2024798373 after retry RPC after error@1285380904) -> (continuation of Deferred@163556269 after retry RPC after error@1217916551) -> (continuation of Deferred@1333419488 after retry RPC after error@1763187516)))
06:33:57.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.590 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@422619657(state=PENDING, result=null, callback=(continuation of Deferred@575713101 after retry RPC after error@1206363067) -> (continuation of Deferred@244391994 after retry RPC after error@309179139) -> (continuation of Deferred@895507652 after retry RPC after error@1331747752) -> (continuation of Deferred@1224188558 after retry RPC after error@930997560) -> (continuation of Deferred@13371759 after retry RPC after error@1236811589) -> (continuation of Deferred@1009649454 after retry RPC after error@1679978934) -> (continuation of Deferred@71666476 after retry RPC after error@210751222), errback=(continuation of Deferred@575713101 after retry RPC after error@1206363067) -> (continuation of Deferred@244391994 after retry RPC after error@309179139) -> (continuation of Deferred@895507652 after retry RPC after error@1331747752) -> (continuation of Deferred@1224188558 after retry RPC after error@930997560) -> (continuation of Deferred@13371759 after retry RPC after error@1236811589) -> (continuation of Deferred@1009649454 after retry RPC after error@1679978934) -> (continuation of Deferred@71666476 after retry RPC after error@210751222)))
06:33:57.831 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@422619657(state=PENDING, result=null, callback=(continuation of Deferred@575713101 after retry RPC after error@1206363067) -> (continuation of Deferred@244391994 after retry RPC after error@309179139) -> (continuation of Deferred@895507652 after retry RPC after error@1331747752) -> (continuation of Deferred@1224188558 after retry RPC after error@930997560) -> (continuation of Deferred@13371759 after retry RPC after error@1236811589) -> (continuation of Deferred@1009649454 after retry RPC after error@1679978934) -> (continuation of Deferred@71666476 after retry RPC after error@210751222), errback=(continuation of Deferred@575713101 after retry RPC after error@1206363067) -> (continuation of Deferred@244391994 after retry RPC after error@309179139) -> (continuation of Deferred@895507652 after retry RPC after error@1331747752) -> (continuation of Deferred@1224188558 after retry RPC after error@930997560) -> (continuation of Deferred@13371759 after retry RPC after error@1236811589) -> (continuation of Deferred@1009649454 after retry RPC after error@1679978934) -> (continuation of Deferred@71666476 after retry RPC after error@210751222)))
06:33:57.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.868 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.868 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.868 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.869 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.930 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.930 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.931 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:57.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:57.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:57.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:57.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:57.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@885980897(state=PENDING, result=null, callback=(continuation of Deferred@379159420 after retry RPC after error@832519735) -> (continuation of Deferred@1503365446 after retry RPC after error@442584418) -> (continuation of Deferred@1085914929 after retry RPC after error@144658327) -> (continuation of Deferred@231421563 after retry RPC after error@491872509) -> (continuation of Deferred@270271569 after retry RPC after error@1653408306) -> (continuation of Deferred@916556022 after retry RPC after error@551632983) -> (continuation of Deferred@1353705110 after retry RPC after error@1522135587), errback=(continuation of Deferred@379159420 after retry RPC after error@832519735) -> (continuation of Deferred@1503365446 after retry RPC after error@442584418) -> (continuation of Deferred@1085914929 after retry RPC after error@144658327) -> (continuation of Deferred@231421563 after retry RPC after error@491872509) -> (continuation of Deferred@270271569 after retry RPC after error@1653408306) -> (continuation of Deferred@916556022 after retry RPC after error@551632983) -> (continuation of Deferred@1353705110 after retry RPC after error@1522135587)))
06:33:58.072 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@885980897(state=PENDING, result=null, callback=(continuation of Deferred@379159420 after retry RPC after error@832519735) -> (continuation of Deferred@1503365446 after retry RPC after error@442584418) -> (continuation of Deferred@1085914929 after retry RPC after error@144658327) -> (continuation of Deferred@231421563 after retry RPC after error@491872509) -> (continuation of Deferred@270271569 after retry RPC after error@1653408306) -> (continuation of Deferred@916556022 after retry RPC after error@551632983) -> (continuation of Deferred@1353705110 after retry RPC after error@1522135587), errback=(continuation of Deferred@379159420 after retry RPC after error@832519735) -> (continuation of Deferred@1503365446 after retry RPC after error@442584418) -> (continuation of Deferred@1085914929 after retry RPC after error@144658327) -> (continuation of Deferred@231421563 after retry RPC after error@491872509) -> (continuation of Deferred@270271569 after retry RPC after error@1653408306) -> (continuation of Deferred@916556022 after retry RPC after error@551632983) -> (continuation of Deferred@1353705110 after retry RPC after error@1522135587)))
06:33:58.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1668977553(state=PENDING, result=null, callback=(continuation of Deferred@1374697056 after retry RPC after error@325042700) -> (continuation of Deferred@1688909185 after retry RPC after error@1408980126) -> (continuation of Deferred@65547191 after retry RPC after error@1850394212) -> (continuation of Deferred@1610440849 after retry RPC after error@1445695572) -> (continuation of Deferred@350006453 after retry RPC after error@740106668) -> (continuation of Deferred@407909718 after retry RPC after error@309575480) -> (continuation of Deferred@1673199815 after retry RPC after error@644767069), errback=(continuation of Deferred@1374697056 after retry RPC after error@325042700) -> (continuation of Deferred@1688909185 after retry RPC after error@1408980126) -> (continuation of Deferred@65547191 after retry RPC after error@1850394212) -> (continuation of Deferred@1610440849 after retry RPC after error@1445695572) -> (continuation of Deferred@350006453 after retry RPC after error@740106668) -> (continuation of Deferred@407909718 after retry RPC after error@309575480) -> (continuation of Deferred@1673199815 after retry RPC after error@644767069)))
06:33:58.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1668977553(state=PENDING, result=null, callback=(continuation of Deferred@1374697056 after retry RPC after error@325042700) -> (continuation of Deferred@1688909185 after retry RPC after error@1408980126) -> (continuation of Deferred@65547191 after retry RPC after error@1850394212) -> (continuation of Deferred@1610440849 after retry RPC after error@1445695572) -> (continuation of Deferred@350006453 after retry RPC after error@740106668) -> (continuation of Deferred@407909718 after retry RPC after error@309575480) -> (continuation of Deferred@1673199815 after retry RPC after error@644767069), errback=(continuation of Deferred@1374697056 after retry RPC after error@325042700) -> (continuation of Deferred@1688909185 after retry RPC after error@1408980126) -> (continuation of Deferred@65547191 after retry RPC after error@1850394212) -> (continuation of Deferred@1610440849 after retry RPC after error@1445695572) -> (continuation of Deferred@350006453 after retry RPC after error@740106668) -> (continuation of Deferred@407909718 after retry RPC after error@309575480) -> (continuation of Deferred@1673199815 after retry RPC after error@644767069)))
06:33:58.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.349 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@353327928(state=PENDING, result=null, callback=(continuation of Deferred@314596599 after retry RPC after error@257795500) -> (continuation of Deferred@1762681891 after retry RPC after error@111748258) -> (continuation of Deferred@371124068 after retry RPC after error@1955348686) -> (continuation of Deferred@509018880 after retry RPC after error@1368696924) -> (continuation of Deferred@818958541 after retry RPC after error@642579497) -> (continuation of Deferred@2467043 after retry RPC after error@1955015411) -> (continuation of Deferred@1189304982 after retry RPC after error@126835671) -> (continuation of Deferred@706798969 after retry RPC after error@1732088444), errback=(continuation of Deferred@314596599 after retry RPC after error@257795500) -> (continuation of Deferred@1762681891 after retry RPC after error@111748258) -> (continuation of Deferred@371124068 after retry RPC after error@1955348686) -> (continuation of Deferred@509018880 after retry RPC after error@1368696924) -> (continuation of Deferred@818958541 after retry RPC after error@642579497) -> (continuation of Deferred@2467043 after retry RPC after error@1955015411) -> (continuation of Deferred@1189304982 after retry RPC after error@126835671) -> (continuation of Deferred@706798969 after retry RPC after error@1732088444)))
06:33:58.491 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@353327928(state=PENDING, result=null, callback=(continuation of Deferred@314596599 after retry RPC after error@257795500) -> (continuation of Deferred@1762681891 after retry RPC after error@111748258) -> (continuation of Deferred@371124068 after retry RPC after error@1955348686) -> (continuation of Deferred@509018880 after retry RPC after error@1368696924) -> (continuation of Deferred@818958541 after retry RPC after error@642579497) -> (continuation of Deferred@2467043 after retry RPC after error@1955015411) -> (continuation of Deferred@1189304982 after retry RPC after error@126835671) -> (continuation of Deferred@706798969 after retry RPC after error@1732088444), errback=(continuation of Deferred@314596599 after retry RPC after error@257795500) -> (continuation of Deferred@1762681891 after retry RPC after error@111748258) -> (continuation of Deferred@371124068 after retry RPC after error@1955348686) -> (continuation of Deferred@509018880 after retry RPC after error@1368696924) -> (continuation of Deferred@818958541 after retry RPC after error@642579497) -> (continuation of Deferred@2467043 after retry RPC after error@1955015411) -> (continuation of Deferred@1189304982 after retry RPC after error@126835671) -> (continuation of Deferred@706798969 after retry RPC after error@1732088444)))
06:33:58.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.509 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.588 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.630 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.632 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@680892614(state=PENDING, result=null, callback=(continuation of Deferred@479160026 after retry RPC after error@249100418) -> (continuation of Deferred@1015788967 after retry RPC after error@945572184) -> (continuation of Deferred@1602375999 after retry RPC after error@383237797) -> (continuation of Deferred@2005927765 after retry RPC after error@769427062) -> (continuation of Deferred@1612329307 after retry RPC after error@1259571563) -> (continuation of Deferred@1020167667 after retry RPC after error@1969773354), errback=(continuation of Deferred@479160026 after retry RPC after error@249100418) -> (continuation of Deferred@1015788967 after retry RPC after error@945572184) -> (continuation of Deferred@1602375999 after retry RPC after error@383237797) -> (continuation of Deferred@2005927765 after retry RPC after error@769427062) -> (continuation of Deferred@1612329307 after retry RPC after error@1259571563) -> (continuation of Deferred@1020167667 after retry RPC after error@1969773354)))
06:33:58.692 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [124ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@680892614(state=PENDING, result=null, callback=(continuation of Deferred@479160026 after retry RPC after error@249100418) -> (continuation of Deferred@1015788967 after retry RPC after error@945572184) -> (continuation of Deferred@1602375999 after retry RPC after error@383237797) -> (continuation of Deferred@2005927765 after retry RPC after error@769427062) -> (continuation of Deferred@1612329307 after retry RPC after error@1259571563) -> (continuation of Deferred@1020167667 after retry RPC after error@1969773354), errback=(continuation of Deferred@479160026 after retry RPC after error@249100418) -> (continuation of Deferred@1015788967 after retry RPC after error@945572184) -> (continuation of Deferred@1602375999 after retry RPC after error@383237797) -> (continuation of Deferred@2005927765 after retry RPC after error@769427062) -> (continuation of Deferred@1612329307 after retry RPC after error@1259571563) -> (continuation of Deferred@1020167667 after retry RPC after error@1969773354)))
06:33:58.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.770 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.770 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.770 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.770 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.771 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@712568837(state=PENDING, result=null, callback=(continuation of Deferred@1083244155 after retry RPC after error@934611311) -> (continuation of Deferred@940492794 after retry RPC after error@1427894310) -> (continuation of Deferred@306167768 after retry RPC after error@232665729) -> (continuation of Deferred@1126241751 after retry RPC after error@1227970389) -> (continuation of Deferred@639623380 after retry RPC after error@1521058988) -> (continuation of Deferred@300789123 after retry RPC after error@1386597140) -> (continuation of Deferred@199596244 after retry RPC after error@494924256), errback=(continuation of Deferred@1083244155 after retry RPC after error@934611311) -> (continuation of Deferred@940492794 after retry RPC after error@1427894310) -> (continuation of Deferred@306167768 after retry RPC after error@232665729) -> (continuation of Deferred@1126241751 after retry RPC after error@1227970389) -> (continuation of Deferred@639623380 after retry RPC after error@1521058988) -> (continuation of Deferred@300789123 after retry RPC after error@1386597140) -> (continuation of Deferred@199596244 after retry RPC after error@494924256)))
06:33:58.952 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@712568837(state=PENDING, result=null, callback=(continuation of Deferred@1083244155 after retry RPC after error@934611311) -> (continuation of Deferred@940492794 after retry RPC after error@1427894310) -> (continuation of Deferred@306167768 after retry RPC after error@232665729) -> (continuation of Deferred@1126241751 after retry RPC after error@1227970389) -> (continuation of Deferred@639623380 after retry RPC after error@1521058988) -> (continuation of Deferred@300789123 after retry RPC after error@1386597140) -> (continuation of Deferred@199596244 after retry RPC after error@494924256), errback=(continuation of Deferred@1083244155 after retry RPC after error@934611311) -> (continuation of Deferred@940492794 after retry RPC after error@1427894310) -> (continuation of Deferred@306167768 after retry RPC after error@232665729) -> (continuation of Deferred@1126241751 after retry RPC after error@1227970389) -> (continuation of Deferred@639623380 after retry RPC after error@1521058988) -> (continuation of Deferred@300789123 after retry RPC after error@1386597140) -> (continuation of Deferred@199596244 after retry RPC after error@494924256)))
06:33:58.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:58.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:58.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:58.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:58.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:58.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.068 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.069 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.110 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@930675371(state=PENDING, result=null, callback=(continuation of Deferred@1759593843 after retry RPC after error@1096141401) -> (continuation of Deferred@194968060 after retry RPC after error@2068047750) -> (continuation of Deferred@434892608 after retry RPC after error@1979971878) -> (continuation of Deferred@170835715 after retry RPC after error@1857361431) -> (continuation of Deferred@685929838 after retry RPC after error@1094808477) -> (continuation of Deferred@1055394321 after retry RPC after error@2024515189) -> (continuation of Deferred@1138353419 after retry RPC after error@156150646), errback=(continuation of Deferred@1759593843 after retry RPC after error@1096141401) -> (continuation of Deferred@194968060 after retry RPC after error@2068047750) -> (continuation of Deferred@434892608 after retry RPC after error@1979971878) -> (continuation of Deferred@170835715 after retry RPC after error@1857361431) -> (continuation of Deferred@685929838 after retry RPC after error@1094808477) -> (continuation of Deferred@1055394321 after retry RPC after error@2024515189) -> (continuation of Deferred@1138353419 after retry RPC after error@156150646)))
06:33:59.111 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@930675371(state=PENDING, result=null, callback=(continuation of Deferred@1759593843 after retry RPC after error@1096141401) -> (continuation of Deferred@194968060 after retry RPC after error@2068047750) -> (continuation of Deferred@434892608 after retry RPC after error@1979971878) -> (continuation of Deferred@170835715 after retry RPC after error@1857361431) -> (continuation of Deferred@685929838 after retry RPC after error@1094808477) -> (continuation of Deferred@1055394321 after retry RPC after error@2024515189) -> (continuation of Deferred@1138353419 after retry RPC after error@156150646), errback=(continuation of Deferred@1759593843 after retry RPC after error@1096141401) -> (continuation of Deferred@194968060 after retry RPC after error@2068047750) -> (continuation of Deferred@434892608 after retry RPC after error@1979971878) -> (continuation of Deferred@170835715 after retry RPC after error@1857361431) -> (continuation of Deferred@685929838 after retry RPC after error@1094808477) -> (continuation of Deferred@1055394321 after retry RPC after error@2024515189) -> (continuation of Deferred@1138353419 after retry RPC after error@156150646)))
06:33:59.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.291 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@329906552(state=PENDING, result=null, callback=(continuation of Deferred@1144700778 after retry RPC after error@759504890) -> (continuation of Deferred@1379024969 after retry RPC after error@235671886) -> (continuation of Deferred@534527498 after retry RPC after error@442714308) -> (continuation of Deferred@1519612873 after retry RPC after error@880512948) -> (continuation of Deferred@1252645727 after retry RPC after error@1250729894) -> (continuation of Deferred@197173253 after retry RPC after error@1672382694), errback=(continuation of Deferred@1144700778 after retry RPC after error@759504890) -> (continuation of Deferred@1379024969 after retry RPC after error@235671886) -> (continuation of Deferred@534527498 after retry RPC after error@442714308) -> (continuation of Deferred@1519612873 after retry RPC after error@880512948) -> (continuation of Deferred@1252645727 after retry RPC after error@1250729894) -> (continuation of Deferred@197173253 after retry RPC after error@1672382694)))
06:33:59.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@329906552(state=PENDING, result=null, callback=(continuation of Deferred@1144700778 after retry RPC after error@759504890) -> (continuation of Deferred@1379024969 after retry RPC after error@235671886) -> (continuation of Deferred@534527498 after retry RPC after error@442714308) -> (continuation of Deferred@1519612873 after retry RPC after error@880512948) -> (continuation of Deferred@1252645727 after retry RPC after error@1250729894) -> (continuation of Deferred@197173253 after retry RPC after error@1672382694), errback=(continuation of Deferred@1144700778 after retry RPC after error@759504890) -> (continuation of Deferred@1379024969 after retry RPC after error@235671886) -> (continuation of Deferred@534527498 after retry RPC after error@442714308) -> (continuation of Deferred@1519612873 after retry RPC after error@880512948) -> (continuation of Deferred@1252645727 after retry RPC after error@1250729894) -> (continuation of Deferred@197173253 after retry RPC after error@1672382694)))
06:33:59.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.410 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.531 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1040948119(state=PENDING, result=null, callback=(continuation of Deferred@1430727128 after retry RPC after error@2025615729) -> (continuation of Deferred@1735363294 after retry RPC after error@89813639) -> (continuation of Deferred@1765769757 after retry RPC after error@1922257330) -> (continuation of Deferred@1773214788 after retry RPC after error@1059689919) -> (continuation of Deferred@705943341 after retry RPC after error@1122307356) -> (continuation of Deferred@592935480 after retry RPC after error@1352701824) -> (continuation of Deferred@1690208262 after retry RPC after error@249164896) -> (continuation of Deferred@382191929 after retry RPC after error@1116688630), errback=(continuation of Deferred@1430727128 after retry RPC after error@2025615729) -> (continuation of Deferred@1735363294 after retry RPC after error@89813639) -> (continuation of Deferred@1765769757 after retry RPC after error@1922257330) -> (continuation of Deferred@1773214788 after retry RPC after error@1059689919) -> (continuation of Deferred@705943341 after retry RPC after error@1122307356) -> (continuation of Deferred@592935480 after retry RPC after error@1352701824) -> (continuation of Deferred@1690208262 after retry RPC after error@249164896) -> (continuation of Deferred@382191929 after retry RPC after error@1116688630)))
06:33:59.532 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1040948119(state=PENDING, result=null, callback=(continuation of Deferred@1430727128 after retry RPC after error@2025615729) -> (continuation of Deferred@1735363294 after retry RPC after error@89813639) -> (continuation of Deferred@1765769757 after retry RPC after error@1922257330) -> (continuation of Deferred@1773214788 after retry RPC after error@1059689919) -> (continuation of Deferred@705943341 after retry RPC after error@1122307356) -> (continuation of Deferred@592935480 after retry RPC after error@1352701824) -> (continuation of Deferred@1690208262 after retry RPC after error@249164896) -> (continuation of Deferred@382191929 after retry RPC after error@1116688630), errback=(continuation of Deferred@1430727128 after retry RPC after error@2025615729) -> (continuation of Deferred@1735363294 after retry RPC after error@89813639) -> (continuation of Deferred@1765769757 after retry RPC after error@1922257330) -> (continuation of Deferred@1773214788 after retry RPC after error@1059689919) -> (continuation of Deferred@705943341 after retry RPC after error@1122307356) -> (continuation of Deferred@592935480 after retry RPC after error@1352701824) -> (continuation of Deferred@1690208262 after retry RPC after error@249164896) -> (continuation of Deferred@382191929 after retry RPC after error@1116688630)))
06:33:59.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.568 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.568 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.628 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.628 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.628 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.629 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.808 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@926661309(state=PENDING, result=null, callback=(continuation of Deferred@388683991 after retry RPC after error@1953261056) -> (continuation of Deferred@1354120461 after retry RPC after error@984783860) -> (continuation of Deferred@2065633301 after retry RPC after error@1825842448) -> (continuation of Deferred@233518034 after retry RPC after error@589853856) -> (continuation of Deferred@540301160 after retry RPC after error@1837439707) -> (continuation of Deferred@958714138 after retry RPC after error@1506357226) -> (continuation of Deferred@1531231404 after retry RPC after error@517088989) -> (continuation of Deferred@23173148 after retry RPC after error@298271141), errback=(continuation of Deferred@388683991 after retry RPC after error@1953261056) -> (continuation of Deferred@1354120461 after retry RPC after error@984783860) -> (continuation of Deferred@2065633301 after retry RPC after error@1825842448) -> (continuation of Deferred@233518034 after retry RPC after error@589853856) -> (continuation of Deferred@540301160 after retry RPC after error@1837439707) -> (continuation of Deferred@958714138 after retry RPC after error@1506357226) -> (continuation of Deferred@1531231404 after retry RPC after error@517088989) -> (continuation of Deferred@23173148 after retry RPC after error@298271141)))
06:33:59.809 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:33:59.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.909 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:33:59.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:33:59.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:33:59.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:33:59.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:33:59.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.010 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.010 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.010 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.011 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1235994417(state=PENDING, result=null, callback=(continuation of Deferred@1802944163 after retry RPC after error@8459760) -> (continuation of Deferred@441916865 after retry RPC after error@223532021) -> (continuation of Deferred@1401943898 after retry RPC after error@1537820238) -> (continuation of Deferred@2033377775 after retry RPC after error@1650743367) -> (continuation of Deferred@1316836982 after retry RPC after error@1610080961) -> (continuation of Deferred@824466761 after retry RPC after error@1900191050), errback=(continuation of Deferred@1802944163 after retry RPC after error@8459760) -> (continuation of Deferred@441916865 after retry RPC after error@223532021) -> (continuation of Deferred@1401943898 after retry RPC after error@1537820238) -> (continuation of Deferred@2033377775 after retry RPC after error@1650743367) -> (continuation of Deferred@1316836982 after retry RPC after error@1610080961) -> (continuation of Deferred@824466761 after retry RPC after error@1900191050)))
06:34:00.012 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1235994417(state=PENDING, result=null, callback=(continuation of Deferred@1802944163 after retry RPC after error@8459760) -> (continuation of Deferred@441916865 after retry RPC after error@223532021) -> (continuation of Deferred@1401943898 after retry RPC after error@1537820238) -> (continuation of Deferred@2033377775 after retry RPC after error@1650743367) -> (continuation of Deferred@1316836982 after retry RPC after error@1610080961) -> (continuation of Deferred@824466761 after retry RPC after error@1900191050), errback=(continuation of Deferred@1802944163 after retry RPC after error@8459760) -> (continuation of Deferred@441916865 after retry RPC after error@223532021) -> (continuation of Deferred@1401943898 after retry RPC after error@1537820238) -> (continuation of Deferred@2033377775 after retry RPC after error@1650743367) -> (continuation of Deferred@1316836982 after retry RPC after error@1610080961) -> (continuation of Deferred@824466761 after retry RPC after error@1900191050)))
06:34:00.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.064 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:00.063930 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:00.066 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:00.066404 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:00.068 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:00.067894 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:00.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.231 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@728658726(state=PENDING, result=null, callback=(continuation of Deferred@1362032476 after retry RPC after error@850601764) -> (continuation of Deferred@1282015743 after retry RPC after error@348498967) -> (continuation of Deferred@855064004 after retry RPC after error@425113273) -> (continuation of Deferred@95365525 after retry RPC after error@1018398755) -> (continuation of Deferred@924945260 after retry RPC after error@460193871) -> (continuation of Deferred@1735483702 after retry RPC after error@1193484139), errback=(continuation of Deferred@1362032476 after retry RPC after error@850601764) -> (continuation of Deferred@1282015743 after retry RPC after error@348498967) -> (continuation of Deferred@855064004 after retry RPC after error@425113273) -> (continuation of Deferred@95365525 after retry RPC after error@1018398755) -> (continuation of Deferred@924945260 after retry RPC after error@460193871) -> (continuation of Deferred@1735483702 after retry RPC after error@1193484139)))
06:34:00.232 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@728658726(state=PENDING, result=null, callback=(continuation of Deferred@1362032476 after retry RPC after error@850601764) -> (continuation of Deferred@1282015743 after retry RPC after error@348498967) -> (continuation of Deferred@855064004 after retry RPC after error@425113273) -> (continuation of Deferred@95365525 after retry RPC after error@1018398755) -> (continuation of Deferred@924945260 after retry RPC after error@460193871) -> (continuation of Deferred@1735483702 after retry RPC after error@1193484139), errback=(continuation of Deferred@1362032476 after retry RPC after error@850601764) -> (continuation of Deferred@1282015743 after retry RPC after error@348498967) -> (continuation of Deferred@855064004 after retry RPC after error@425113273) -> (continuation of Deferred@95365525 after retry RPC after error@1018398755) -> (continuation of Deferred@924945260 after retry RPC after error@460193871) -> (continuation of Deferred@1735483702 after retry RPC after error@1193484139)))
06:34:00.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.430 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1011439564(state=PENDING, result=null, callback=(continuation of Deferred@991193259 after retry RPC after error@1025387035) -> (continuation of Deferred@1185510156 after retry RPC after error@807971618) -> (continuation of Deferred@677502140 after retry RPC after error@1684239705) -> (continuation of Deferred@1776241610 after retry RPC after error@475761768) -> (continuation of Deferred@981018467 after retry RPC after error@1162826952) -> (continuation of Deferred@1448990384 after retry RPC after error@371534047), errback=(continuation of Deferred@991193259 after retry RPC after error@1025387035) -> (continuation of Deferred@1185510156 after retry RPC after error@807971618) -> (continuation of Deferred@677502140 after retry RPC after error@1684239705) -> (continuation of Deferred@1776241610 after retry RPC after error@475761768) -> (continuation of Deferred@981018467 after retry RPC after error@1162826952) -> (continuation of Deferred@1448990384 after retry RPC after error@371534047)))
06:34:00.431 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1011439564(state=PENDING, result=null, callback=(continuation of Deferred@991193259 after retry RPC after error@1025387035) -> (continuation of Deferred@1185510156 after retry RPC after error@807971618) -> (continuation of Deferred@677502140 after retry RPC after error@1684239705) -> (continuation of Deferred@1776241610 after retry RPC after error@475761768) -> (continuation of Deferred@981018467 after retry RPC after error@1162826952) -> (continuation of Deferred@1448990384 after retry RPC after error@371534047), errback=(continuation of Deferred@991193259 after retry RPC after error@1025387035) -> (continuation of Deferred@1185510156 after retry RPC after error@807971618) -> (continuation of Deferred@677502140 after retry RPC after error@1684239705) -> (continuation of Deferred@1776241610 after retry RPC after error@475761768) -> (continuation of Deferred@981018467 after retry RPC after error@1162826952) -> (continuation of Deferred@1448990384 after retry RPC after error@371534047)))
06:34:00.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.690 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.690 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.691 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.709 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@510146053(state=PENDING, result=null, callback=(continuation of Deferred@26272154 after retry RPC after error@1382896459) -> (continuation of Deferred@886638548 after retry RPC after error@1809762132) -> (continuation of Deferred@1833203588 after retry RPC after error@1435211372) -> (continuation of Deferred@697355438 after retry RPC after error@879315631) -> (continuation of Deferred@2053104625 after retry RPC after error@1944990240) -> (continuation of Deferred@1555330383 after retry RPC after error@586571844) -> (continuation of Deferred@666066816 after retry RPC after error@634259044) -> (continuation of Deferred@1521418655 after retry RPC after error@1263185614), errback=(continuation of Deferred@26272154 after retry RPC after error@1382896459) -> (continuation of Deferred@886638548 after retry RPC after error@1809762132) -> (continuation of Deferred@1833203588 after retry RPC after error@1435211372) -> (continuation of Deferred@697355438 after retry RPC after error@879315631) -> (continuation of Deferred@2053104625 after retry RPC after error@1944990240) -> (continuation of Deferred@1555330383 after retry RPC after error@586571844) -> (continuation of Deferred@666066816 after retry RPC after error@634259044) -> (continuation of Deferred@1521418655 after retry RPC after error@1263185614)))
06:34:00.709 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:00.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.810 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@483019115(state=PENDING, result=null, callback=(continuation of Deferred@2059836330 after retry RPC after error@2045968507) -> (continuation of Deferred@1307895365 after retry RPC after error@62619521) -> (continuation of Deferred@1307492247 after retry RPC after error@754945673) -> (continuation of Deferred@1188266909 after retry RPC after error@516729777) -> (continuation of Deferred@644263416 after retry RPC after error@1983767675) -> (continuation of Deferred@749231690 after retry RPC after error@1472395597) -> (continuation of Deferred@1155045152 after retry RPC after error@1574944252) -> (continuation of Deferred@862708227 after retry RPC after error@1931701682), errback=(continuation of Deferred@2059836330 after retry RPC after error@2045968507) -> (continuation of Deferred@1307895365 after retry RPC after error@62619521) -> (continuation of Deferred@1307492247 after retry RPC after error@754945673) -> (continuation of Deferred@1188266909 after retry RPC after error@516729777) -> (continuation of Deferred@644263416 after retry RPC after error@1983767675) -> (continuation of Deferred@749231690 after retry RPC after error@1472395597) -> (continuation of Deferred@1155045152 after retry RPC after error@1574944252) -> (continuation of Deferred@862708227 after retry RPC after error@1931701682)))
06:34:00.952 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@483019115(state=PENDING, result=null, callback=(continuation of Deferred@2059836330 after retry RPC after error@2045968507) -> (continuation of Deferred@1307895365 after retry RPC after error@62619521) -> (continuation of Deferred@1307492247 after retry RPC after error@754945673) -> (continuation of Deferred@1188266909 after retry RPC after error@516729777) -> (continuation of Deferred@644263416 after retry RPC after error@1983767675) -> (continuation of Deferred@749231690 after retry RPC after error@1472395597) -> (continuation of Deferred@1155045152 after retry RPC after error@1574944252) -> (continuation of Deferred@862708227 after retry RPC after error@1931701682), errback=(continuation of Deferred@2059836330 after retry RPC after error@2045968507) -> (continuation of Deferred@1307895365 after retry RPC after error@62619521) -> (continuation of Deferred@1307492247 after retry RPC after error@754945673) -> (continuation of Deferred@1188266909 after retry RPC after error@516729777) -> (continuation of Deferred@644263416 after retry RPC after error@1983767675) -> (continuation of Deferred@749231690 after retry RPC after error@1472395597) -> (continuation of Deferred@1155045152 after retry RPC after error@1574944252) -> (continuation of Deferred@862708227 after retry RPC after error@1931701682)))
06:34:00.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:00.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:00.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:00.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:00.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:00.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.008 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.009 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.150 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@159936370(state=PENDING, result=null, callback=(continuation of Deferred@450792536 after retry RPC after error@1169342623) -> (continuation of Deferred@483220150 after retry RPC after error@1458853282) -> (continuation of Deferred@950452072 after retry RPC after error@1807955675) -> (continuation of Deferred@497932799 after retry RPC after error@1770051245) -> (continuation of Deferred@1691379978 after retry RPC after error@374861477) -> (continuation of Deferred@1707519921 after retry RPC after error@169621530) -> (continuation of Deferred@1843632234 after retry RPC after error@1147695541), errback=(continuation of Deferred@450792536 after retry RPC after error@1169342623) -> (continuation of Deferred@483220150 after retry RPC after error@1458853282) -> (continuation of Deferred@950452072 after retry RPC after error@1807955675) -> (continuation of Deferred@497932799 after retry RPC after error@1770051245) -> (continuation of Deferred@1691379978 after retry RPC after error@374861477) -> (continuation of Deferred@1707519921 after retry RPC after error@169621530) -> (continuation of Deferred@1843632234 after retry RPC after error@1147695541)))
06:34:01.191 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [183ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@159936370(state=PENDING, result=null, callback=(continuation of Deferred@450792536 after retry RPC after error@1169342623) -> (continuation of Deferred@483220150 after retry RPC after error@1458853282) -> (continuation of Deferred@950452072 after retry RPC after error@1807955675) -> (continuation of Deferred@497932799 after retry RPC after error@1770051245) -> (continuation of Deferred@1691379978 after retry RPC after error@374861477) -> (continuation of Deferred@1707519921 after retry RPC after error@169621530) -> (continuation of Deferred@1843632234 after retry RPC after error@1147695541), errback=(continuation of Deferred@450792536 after retry RPC after error@1169342623) -> (continuation of Deferred@483220150 after retry RPC after error@1458853282) -> (continuation of Deferred@950452072 after retry RPC after error@1807955675) -> (continuation of Deferred@497932799 after retry RPC after error@1770051245) -> (continuation of Deferred@1691379978 after retry RPC after error@374861477) -> (continuation of Deferred@1707519921 after retry RPC after error@169621530) -> (continuation of Deferred@1843632234 after retry RPC after error@1147695541)))
06:34:01.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@440314572(state=PENDING, result=null, callback=(continuation of Deferred@1524737624 after retry RPC after error@1913932331) -> (continuation of Deferred@1722870907 after retry RPC after error@163009195) -> (continuation of Deferred@1588852387 after retry RPC after error@237414988) -> (continuation of Deferred@1871994055 after retry RPC after error@1552247188) -> (continuation of Deferred@490904137 after retry RPC after error@646784871) -> (continuation of Deferred@1806837013 after retry RPC after error@478951486) -> (continuation of Deferred@353728820 after retry RPC after error@1517081326), errback=(continuation of Deferred@1524737624 after retry RPC after error@1913932331) -> (continuation of Deferred@1722870907 after retry RPC after error@163009195) -> (continuation of Deferred@1588852387 after retry RPC after error@237414988) -> (continuation of Deferred@1871994055 after retry RPC after error@1552247188) -> (continuation of Deferred@490904137 after retry RPC after error@646784871) -> (continuation of Deferred@1806837013 after retry RPC after error@478951486) -> (continuation of Deferred@353728820 after retry RPC after error@1517081326)))
06:34:01.451 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@440314572(state=PENDING, result=null, callback=(continuation of Deferred@1524737624 after retry RPC after error@1913932331) -> (continuation of Deferred@1722870907 after retry RPC after error@163009195) -> (continuation of Deferred@1588852387 after retry RPC after error@237414988) -> (continuation of Deferred@1871994055 after retry RPC after error@1552247188) -> (continuation of Deferred@490904137 after retry RPC after error@646784871) -> (continuation of Deferred@1806837013 after retry RPC after error@478951486) -> (continuation of Deferred@353728820 after retry RPC after error@1517081326), errback=(continuation of Deferred@1524737624 after retry RPC after error@1913932331) -> (continuation of Deferred@1722870907 after retry RPC after error@163009195) -> (continuation of Deferred@1588852387 after retry RPC after error@237414988) -> (continuation of Deferred@1871994055 after retry RPC after error@1552247188) -> (continuation of Deferred@490904137 after retry RPC after error@646784871) -> (continuation of Deferred@1806837013 after retry RPC after error@478951486) -> (continuation of Deferred@353728820 after retry RPC after error@1517081326)))
06:34:01.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.470 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.528 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.711 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [59ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1706341972(state=PENDING, result=null, callback=(continuation of Deferred@1925662127 after retry RPC after error@1535341433) -> (continuation of Deferred@1817738478 after retry RPC after error@748577063) -> (continuation of Deferred@1690084134 after retry RPC after error@610536563) -> (continuation of Deferred@192195357 after retry RPC after error@1421632525) -> (continuation of Deferred@1436047407 after retry RPC after error@1360777595) -> (continuation of Deferred@2140779526 after retry RPC after error@1219640308) -> (continuation of Deferred@1610370029 after retry RPC after error@1357109600), errback=(continuation of Deferred@1925662127 after retry RPC after error@1535341433) -> (continuation of Deferred@1817738478 after retry RPC after error@748577063) -> (continuation of Deferred@1690084134 after retry RPC after error@610536563) -> (continuation of Deferred@192195357 after retry RPC after error@1421632525) -> (continuation of Deferred@1436047407 after retry RPC after error@1360777595) -> (continuation of Deferred@2140779526 after retry RPC after error@1219640308) -> (continuation of Deferred@1610370029 after retry RPC after error@1357109600)))
06:34:01.711 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [59ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1706341972(state=PENDING, result=null, callback=(continuation of Deferred@1925662127 after retry RPC after error@1535341433) -> (continuation of Deferred@1817738478 after retry RPC after error@748577063) -> (continuation of Deferred@1690084134 after retry RPC after error@610536563) -> (continuation of Deferred@192195357 after retry RPC after error@1421632525) -> (continuation of Deferred@1436047407 after retry RPC after error@1360777595) -> (continuation of Deferred@2140779526 after retry RPC after error@1219640308) -> (continuation of Deferred@1610370029 after retry RPC after error@1357109600), errback=(continuation of Deferred@1925662127 after retry RPC after error@1535341433) -> (continuation of Deferred@1817738478 after retry RPC after error@748577063) -> (continuation of Deferred@1690084134 after retry RPC after error@610536563) -> (continuation of Deferred@192195357 after retry RPC after error@1421632525) -> (continuation of Deferred@1436047407 after retry RPC after error@1360777595) -> (continuation of Deferred@2140779526 after retry RPC after error@1219640308) -> (continuation of Deferred@1610370029 after retry RPC after error@1357109600)))
06:34:01.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.808 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1682882382(state=PENDING, result=null, callback=(continuation of Deferred@1378015514 after retry RPC after error@1736707273) -> (continuation of Deferred@294952721 after retry RPC after error@3270306) -> (continuation of Deferred@1142883767 after retry RPC after error@62569401) -> (continuation of Deferred@912511176 after retry RPC after error@971514960) -> (continuation of Deferred@99007591 after retry RPC after error@1406646289) -> (continuation of Deferred@2065356595 after retry RPC after error@52995748) -> (continuation of Deferred@865937186 after retry RPC after error@1575229040) -> (continuation of Deferred@1886052784 after retry RPC after error@909206687), errback=(continuation of Deferred@1378015514 after retry RPC after error@1736707273) -> (continuation of Deferred@294952721 after retry RPC after error@3270306) -> (continuation of Deferred@1142883767 after retry RPC after error@62569401) -> (continuation of Deferred@912511176 after retry RPC after error@971514960) -> (continuation of Deferred@99007591 after retry RPC after error@1406646289) -> (continuation of Deferred@2065356595 after retry RPC after error@52995748) -> (continuation of Deferred@865937186 after retry RPC after error@1575229040) -> (continuation of Deferred@1886052784 after retry RPC after error@909206687)))
06:34:01.951 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1682882382(state=PENDING, result=null, callback=(continuation of Deferred@1378015514 after retry RPC after error@1736707273) -> (continuation of Deferred@294952721 after retry RPC after error@3270306) -> (continuation of Deferred@1142883767 after retry RPC after error@62569401) -> (continuation of Deferred@912511176 after retry RPC after error@971514960) -> (continuation of Deferred@99007591 after retry RPC after error@1406646289) -> (continuation of Deferred@2065356595 after retry RPC after error@52995748) -> (continuation of Deferred@865937186 after retry RPC after error@1575229040) -> (continuation of Deferred@1886052784 after retry RPC after error@909206687), errback=(continuation of Deferred@1378015514 after retry RPC after error@1736707273) -> (continuation of Deferred@294952721 after retry RPC after error@3270306) -> (continuation of Deferred@1142883767 after retry RPC after error@62569401) -> (continuation of Deferred@912511176 after retry RPC after error@971514960) -> (continuation of Deferred@99007591 after retry RPC after error@1406646289) -> (continuation of Deferred@2065356595 after retry RPC after error@52995748) -> (continuation of Deferred@865937186 after retry RPC after error@1575229040) -> (continuation of Deferred@1886052784 after retry RPC after error@909206687)))
06:34:01.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:01.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:01.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:01.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:01.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:01.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.090 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@291902282(state=PENDING, result=null, callback=(continuation of Deferred@1894659675 after retry RPC after error@1768926255) -> (continuation of Deferred@1377111549 after retry RPC after error@1942530230) -> (continuation of Deferred@1913980026 after retry RPC after error@1712312515) -> (continuation of Deferred@470762266 after retry RPC after error@708105525) -> (continuation of Deferred@1893212893 after retry RPC after error@1749642701) -> (continuation of Deferred@238609485 after retry RPC after error@44631490) -> (continuation of Deferred@869663104 after retry RPC after error@282211634) -> (continuation of Deferred@1493283305 after retry RPC after error@332146718), errback=(continuation of Deferred@1894659675 after retry RPC after error@1768926255) -> (continuation of Deferred@1377111549 after retry RPC after error@1942530230) -> (continuation of Deferred@1913980026 after retry RPC after error@1712312515) -> (continuation of Deferred@470762266 after retry RPC after error@708105525) -> (continuation of Deferred@1893212893 after retry RPC after error@1749642701) -> (continuation of Deferred@238609485 after retry RPC after error@44631490) -> (continuation of Deferred@869663104 after retry RPC after error@282211634) -> (continuation of Deferred@1493283305 after retry RPC after error@332146718)))
06:34:02.191 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@291902282(state=PENDING, result=null, callback=(continuation of Deferred@1894659675 after retry RPC after error@1768926255) -> (continuation of Deferred@1377111549 after retry RPC after error@1942530230) -> (continuation of Deferred@1913980026 after retry RPC after error@1712312515) -> (continuation of Deferred@470762266 after retry RPC after error@708105525) -> (continuation of Deferred@1893212893 after retry RPC after error@1749642701) -> (continuation of Deferred@238609485 after retry RPC after error@44631490) -> (continuation of Deferred@869663104 after retry RPC after error@282211634) -> (continuation of Deferred@1493283305 after retry RPC after error@332146718), errback=(continuation of Deferred@1894659675 after retry RPC after error@1768926255) -> (continuation of Deferred@1377111549 after retry RPC after error@1942530230) -> (continuation of Deferred@1913980026 after retry RPC after error@1712312515) -> (continuation of Deferred@470762266 after retry RPC after error@708105525) -> (continuation of Deferred@1893212893 after retry RPC after error@1749642701) -> (continuation of Deferred@238609485 after retry RPC after error@44631490) -> (continuation of Deferred@869663104 after retry RPC after error@282211634) -> (continuation of Deferred@1493283305 after retry RPC after error@332146718)))
06:34:02.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.289 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:02.320524 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:02.320 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:02.320748 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:02.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.330 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.331 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.410 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.410 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.411 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.411 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.411 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.412 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1557421874(state=PENDING, result=null, callback=(continuation of Deferred@1246026384 after retry RPC after error@1400678533) -> (continuation of Deferred@1701785543 after retry RPC after error@456970647) -> (continuation of Deferred@2000685114 after retry RPC after error@1095679703) -> (continuation of Deferred@1573105710 after retry RPC after error@537817704) -> (continuation of Deferred@1467531147 after retry RPC after error@1676942261) -> (continuation of Deferred@594969893 after retry RPC after error@1882730001), errback=(continuation of Deferred@1246026384 after retry RPC after error@1400678533) -> (continuation of Deferred@1701785543 after retry RPC after error@456970647) -> (continuation of Deferred@2000685114 after retry RPC after error@1095679703) -> (continuation of Deferred@1573105710 after retry RPC after error@537817704) -> (continuation of Deferred@1467531147 after retry RPC after error@1676942261) -> (continuation of Deferred@594969893 after retry RPC after error@1882730001)))
06:34:02.412 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1557421874(state=PENDING, result=null, callback=(continuation of Deferred@1246026384 after retry RPC after error@1400678533) -> (continuation of Deferred@1701785543 after retry RPC after error@456970647) -> (continuation of Deferred@2000685114 after retry RPC after error@1095679703) -> (continuation of Deferred@1573105710 after retry RPC after error@537817704) -> (continuation of Deferred@1467531147 after retry RPC after error@1676942261) -> (continuation of Deferred@594969893 after retry RPC after error@1882730001), errback=(continuation of Deferred@1246026384 after retry RPC after error@1400678533) -> (continuation of Deferred@1701785543 after retry RPC after error@456970647) -> (continuation of Deferred@2000685114 after retry RPC after error@1095679703) -> (continuation of Deferred@1573105710 after retry RPC after error@537817704) -> (continuation of Deferred@1467531147 after retry RPC after error@1676942261) -> (continuation of Deferred@594969893 after retry RPC after error@1882730001)))
06:34:02.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.448 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.448 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.591 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1869307273(state=PENDING, result=null, callback=(continuation of Deferred@682435614 after retry RPC after error@127507597) -> (continuation of Deferred@162833016 after retry RPC after error@1007038930) -> (continuation of Deferred@2102030232 after retry RPC after error@922548552) -> (continuation of Deferred@452409997 after retry RPC after error@393347232) -> (continuation of Deferred@971672441 after retry RPC after error@1745683381) -> (continuation of Deferred@1259171691 after retry RPC after error@1283260842), errback=(continuation of Deferred@682435614 after retry RPC after error@127507597) -> (continuation of Deferred@162833016 after retry RPC after error@1007038930) -> (continuation of Deferred@2102030232 after retry RPC after error@922548552) -> (continuation of Deferred@452409997 after retry RPC after error@393347232) -> (continuation of Deferred@971672441 after retry RPC after error@1745683381) -> (continuation of Deferred@1259171691 after retry RPC after error@1283260842)))
06:34:02.592 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1869307273(state=PENDING, result=null, callback=(continuation of Deferred@682435614 after retry RPC after error@127507597) -> (continuation of Deferred@162833016 after retry RPC after error@1007038930) -> (continuation of Deferred@2102030232 after retry RPC after error@922548552) -> (continuation of Deferred@452409997 after retry RPC after error@393347232) -> (continuation of Deferred@971672441 after retry RPC after error@1745683381) -> (continuation of Deferred@1259171691 after retry RPC after error@1283260842), errback=(continuation of Deferred@682435614 after retry RPC after error@127507597) -> (continuation of Deferred@162833016 after retry RPC after error@1007038930) -> (continuation of Deferred@2102030232 after retry RPC after error@922548552) -> (continuation of Deferred@452409997 after retry RPC after error@393347232) -> (continuation of Deferred@971672441 after retry RPC after error@1745683381) -> (continuation of Deferred@1259171691 after retry RPC after error@1283260842)))
06:34:02.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.650 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1520396168(state=PENDING, result=null, callback=(continuation of Deferred@258356017 after retry RPC after error@1881974760) -> (continuation of Deferred@119564369 after retry RPC after error@639822874) -> (continuation of Deferred@409189460 after retry RPC after error@224742231) -> (continuation of Deferred@151242826 after retry RPC after error@955299660) -> (continuation of Deferred@1031027994 after retry RPC after error@443344400) -> (continuation of Deferred@221916303 after retry RPC after error@810393779) -> (continuation of Deferred@751683405 after retry RPC after error@1366091938), errback=(continuation of Deferred@258356017 after retry RPC after error@1881974760) -> (continuation of Deferred@119564369 after retry RPC after error@639822874) -> (continuation of Deferred@409189460 after retry RPC after error@224742231) -> (continuation of Deferred@151242826 after retry RPC after error@955299660) -> (continuation of Deferred@1031027994 after retry RPC after error@443344400) -> (continuation of Deferred@221916303 after retry RPC after error@810393779) -> (continuation of Deferred@751683405 after retry RPC after error@1366091938)))
06:34:02.831 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1520396168(state=PENDING, result=null, callback=(continuation of Deferred@258356017 after retry RPC after error@1881974760) -> (continuation of Deferred@119564369 after retry RPC after error@639822874) -> (continuation of Deferred@409189460 after retry RPC after error@224742231) -> (continuation of Deferred@151242826 after retry RPC after error@955299660) -> (continuation of Deferred@1031027994 after retry RPC after error@443344400) -> (continuation of Deferred@221916303 after retry RPC after error@810393779) -> (continuation of Deferred@751683405 after retry RPC after error@1366091938), errback=(continuation of Deferred@258356017 after retry RPC after error@1881974760) -> (continuation of Deferred@119564369 after retry RPC after error@639822874) -> (continuation of Deferred@409189460 after retry RPC after error@224742231) -> (continuation of Deferred@151242826 after retry RPC after error@955299660) -> (continuation of Deferred@1031027994 after retry RPC after error@443344400) -> (continuation of Deferred@221916303 after retry RPC after error@810393779) -> (continuation of Deferred@751683405 after retry RPC after error@1366091938)))
06:34:02.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.848 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.848 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.849 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.889 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:02.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:02.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:02.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:02.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:02.991 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1382880024(state=PENDING, result=null, callback=(continuation of Deferred@1606353807 after retry RPC after error@1219005271) -> (continuation of Deferred@1146821723 after retry RPC after error@2115248667) -> (continuation of Deferred@1494283565 after retry RPC after error@1433284459) -> (continuation of Deferred@1238063193 after retry RPC after error@370192257) -> (continuation of Deferred@1224186355 after retry RPC after error@1824973553) -> (continuation of Deferred@2145150624 after retry RPC after error@1822667389), errback=(continuation of Deferred@1606353807 after retry RPC after error@1219005271) -> (continuation of Deferred@1146821723 after retry RPC after error@2115248667) -> (continuation of Deferred@1494283565 after retry RPC after error@1433284459) -> (continuation of Deferred@1238063193 after retry RPC after error@370192257) -> (continuation of Deferred@1224186355 after retry RPC after error@1824973553) -> (continuation of Deferred@2145150624 after retry RPC after error@1822667389)))
06:34:02.992 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=143), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1382880024(state=PENDING, result=null, callback=(continuation of Deferred@1606353807 after retry RPC after error@1219005271) -> (continuation of Deferred@1146821723 after retry RPC after error@2115248667) -> (continuation of Deferred@1494283565 after retry RPC after error@1433284459) -> (continuation of Deferred@1238063193 after retry RPC after error@370192257) -> (continuation of Deferred@1224186355 after retry RPC after error@1824973553) -> (continuation of Deferred@2145150624 after retry RPC after error@1822667389), errback=(continuation of Deferred@1606353807 after retry RPC after error@1219005271) -> (continuation of Deferred@1146821723 after retry RPC after error@2115248667) -> (continuation of Deferred@1494283565 after retry RPC after error@1433284459) -> (continuation of Deferred@1238063193 after retry RPC after error@370192257) -> (continuation of Deferred@1224186355 after retry RPC after error@1824973553) -> (continuation of Deferred@2145150624 after retry RPC after error@1822667389)))
06:34:03.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.048 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.048 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.049 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.110 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1860846283(state=PENDING, result=null, callback=(continuation of Deferred@1890676414 after retry RPC after error@742832419) -> (continuation of Deferred@1921747740 after retry RPC after error@1313563548) -> (continuation of Deferred@896807029 after retry RPC after error@753010777) -> (continuation of Deferred@35628466 after retry RPC after error@391995886) -> (continuation of Deferred@62790888 after retry RPC after error@1067609094) -> (continuation of Deferred@1089432693 after retry RPC after error@1019291543) -> (continuation of Deferred@1340810683 after retry RPC after error@1661317660), errback=(continuation of Deferred@1890676414 after retry RPC after error@742832419) -> (continuation of Deferred@1921747740 after retry RPC after error@1313563548) -> (continuation of Deferred@896807029 after retry RPC after error@753010777) -> (continuation of Deferred@35628466 after retry RPC after error@391995886) -> (continuation of Deferred@62790888 after retry RPC after error@1067609094) -> (continuation of Deferred@1089432693 after retry RPC after error@1019291543) -> (continuation of Deferred@1340810683 after retry RPC after error@1661317660)))
06:34:03.191 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1860846283(state=PENDING, result=null, callback=(continuation of Deferred@1890676414 after retry RPC after error@742832419) -> (continuation of Deferred@1921747740 after retry RPC after error@1313563548) -> (continuation of Deferred@896807029 after retry RPC after error@753010777) -> (continuation of Deferred@35628466 after retry RPC after error@391995886) -> (continuation of Deferred@62790888 after retry RPC after error@1067609094) -> (continuation of Deferred@1089432693 after retry RPC after error@1019291543) -> (continuation of Deferred@1340810683 after retry RPC after error@1661317660), errback=(continuation of Deferred@1890676414 after retry RPC after error@742832419) -> (continuation of Deferred@1921747740 after retry RPC after error@1313563548) -> (continuation of Deferred@896807029 after retry RPC after error@753010777) -> (continuation of Deferred@35628466 after retry RPC after error@391995886) -> (continuation of Deferred@62790888 after retry RPC after error@1067609094) -> (continuation of Deferred@1089432693 after retry RPC after error@1019291543) -> (continuation of Deferred@1340810683 after retry RPC after error@1661317660)))
06:34:03.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.452 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [119ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@913500899(state=PENDING, result=null, callback=(continuation of Deferred@1345920252 after retry RPC after error@1970881556) -> (continuation of Deferred@957136136 after retry RPC after error@403205432) -> (continuation of Deferred@13519929 after retry RPC after error@1059647952) -> (continuation of Deferred@1670447040 after retry RPC after error@950677287) -> (continuation of Deferred@1249317775 after retry RPC after error@841604448) -> (continuation of Deferred@1445153306 after retry RPC after error@1058637038) -> (continuation of Deferred@1300239337 after retry RPC after error@826152420), errback=(continuation of Deferred@1345920252 after retry RPC after error@1970881556) -> (continuation of Deferred@957136136 after retry RPC after error@403205432) -> (continuation of Deferred@13519929 after retry RPC after error@1059647952) -> (continuation of Deferred@1670447040 after retry RPC after error@950677287) -> (continuation of Deferred@1249317775 after retry RPC after error@841604448) -> (continuation of Deferred@1445153306 after retry RPC after error@1058637038) -> (continuation of Deferred@1300239337 after retry RPC after error@826152420)))
06:34:03.452 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [119ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@913500899(state=PENDING, result=null, callback=(continuation of Deferred@1345920252 after retry RPC after error@1970881556) -> (continuation of Deferred@957136136 after retry RPC after error@403205432) -> (continuation of Deferred@13519929 after retry RPC after error@1059647952) -> (continuation of Deferred@1670447040 after retry RPC after error@950677287) -> (continuation of Deferred@1249317775 after retry RPC after error@841604448) -> (continuation of Deferred@1445153306 after retry RPC after error@1058637038) -> (continuation of Deferred@1300239337 after retry RPC after error@826152420), errback=(continuation of Deferred@1345920252 after retry RPC after error@1970881556) -> (continuation of Deferred@957136136 after retry RPC after error@403205432) -> (continuation of Deferred@13519929 after retry RPC after error@1059647952) -> (continuation of Deferred@1670447040 after retry RPC after error@950677287) -> (continuation of Deferred@1249317775 after retry RPC after error@841604448) -> (continuation of Deferred@1445153306 after retry RPC after error@1058637038) -> (continuation of Deferred@1300239337 after retry RPC after error@826152420)))
06:34:03.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.729 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1074821415(state=PENDING, result=null, callback=(continuation of Deferred@1443728949 after retry RPC after error@1101688387) -> (continuation of Deferred@1507765223 after retry RPC after error@449063098) -> (continuation of Deferred@545419761 after retry RPC after error@852024276) -> (continuation of Deferred@509992953 after retry RPC after error@635417975) -> (continuation of Deferred@383866308 after retry RPC after error@1644374711) -> (continuation of Deferred@1378462489 after retry RPC after error@168151078) -> (continuation of Deferred@383181783 after retry RPC after error@1845175592), errback=(continuation of Deferred@1443728949 after retry RPC after error@1101688387) -> (continuation of Deferred@1507765223 after retry RPC after error@449063098) -> (continuation of Deferred@545419761 after retry RPC after error@852024276) -> (continuation of Deferred@509992953 after retry RPC after error@635417975) -> (continuation of Deferred@383866308 after retry RPC after error@1644374711) -> (continuation of Deferred@1378462489 after retry RPC after error@168151078) -> (continuation of Deferred@383181783 after retry RPC after error@1845175592)))
06:34:03.729 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:03.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:03.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@98906762(state=PENDING, result=null, callback=(continuation of Deferred@475073239 after retry RPC after error@1320136528) -> (continuation of Deferred@1377971246 after retry RPC after error@2130654594) -> (continuation of Deferred@1785503544 after retry RPC after error@1493854218) -> (continuation of Deferred@436890713 after retry RPC after error@492702143) -> (continuation of Deferred@331731059 after retry RPC after error@419803193) -> (continuation of Deferred@1352957924 after retry RPC after error@1037731774) -> (continuation of Deferred@2203198 after retry RPC after error@881943163), errback=(continuation of Deferred@475073239 after retry RPC after error@1320136528) -> (continuation of Deferred@1377971246 after retry RPC after error@2130654594) -> (continuation of Deferred@1785503544 after retry RPC after error@1493854218) -> (continuation of Deferred@436890713 after retry RPC after error@492702143) -> (continuation of Deferred@331731059 after retry RPC after error@419803193) -> (continuation of Deferred@1352957924 after retry RPC after error@1037731774) -> (continuation of Deferred@2203198 after retry RPC after error@881943163)))
06:34:03.971 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@98906762(state=PENDING, result=null, callback=(continuation of Deferred@475073239 after retry RPC after error@1320136528) -> (continuation of Deferred@1377971246 after retry RPC after error@2130654594) -> (continuation of Deferred@1785503544 after retry RPC after error@1493854218) -> (continuation of Deferred@436890713 after retry RPC after error@492702143) -> (continuation of Deferred@331731059 after retry RPC after error@419803193) -> (continuation of Deferred@1352957924 after retry RPC after error@1037731774) -> (continuation of Deferred@2203198 after retry RPC after error@881943163), errback=(continuation of Deferred@475073239 after retry RPC after error@1320136528) -> (continuation of Deferred@1377971246 after retry RPC after error@2130654594) -> (continuation of Deferred@1785503544 after retry RPC after error@1493854218) -> (continuation of Deferred@436890713 after retry RPC after error@492702143) -> (continuation of Deferred@331731059 after retry RPC after error@419803193) -> (continuation of Deferred@1352957924 after retry RPC after error@1037731774) -> (continuation of Deferred@2203198 after retry RPC after error@881943163)))
06:34:03.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:03.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:03.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:03.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:03.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:04.015331 25652 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:34:04.015 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:04.015594 25652 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:34:04.016 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:04.016558 25652 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:34:04.017 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:04.017613 25333 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:34:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:04.018888 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:34:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:04.019063 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:34:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:04.019178 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:34:04.019 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:04.019376 25652 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:34:04.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.190 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@2046783006(state=PENDING, result=null, callback=(continuation of Deferred@1230994456 after retry RPC after error@858279852) -> (continuation of Deferred@1017082620 after retry RPC after error@1049785583) -> (continuation of Deferred@211775641 after retry RPC after error@2109414281) -> (continuation of Deferred@989109840 after retry RPC after error@392999149) -> (continuation of Deferred@1610617121 after retry RPC after error@1161964027) -> (continuation of Deferred@1532639517 after retry RPC after error@700645699), errback=(continuation of Deferred@1230994456 after retry RPC after error@858279852) -> (continuation of Deferred@1017082620 after retry RPC after error@1049785583) -> (continuation of Deferred@211775641 after retry RPC after error@2109414281) -> (continuation of Deferred@989109840 after retry RPC after error@392999149) -> (continuation of Deferred@1610617121 after retry RPC after error@1161964027) -> (continuation of Deferred@1532639517 after retry RPC after error@700645699)))
06:34:04.191 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@2046783006(state=PENDING, result=null, callback=(continuation of Deferred@1230994456 after retry RPC after error@858279852) -> (continuation of Deferred@1017082620 after retry RPC after error@1049785583) -> (continuation of Deferred@211775641 after retry RPC after error@2109414281) -> (continuation of Deferred@989109840 after retry RPC after error@392999149) -> (continuation of Deferred@1610617121 after retry RPC after error@1161964027) -> (continuation of Deferred@1532639517 after retry RPC after error@700645699), errback=(continuation of Deferred@1230994456 after retry RPC after error@858279852) -> (continuation of Deferred@1017082620 after retry RPC after error@1049785583) -> (continuation of Deferred@211775641 after retry RPC after error@2109414281) -> (continuation of Deferred@989109840 after retry RPC after error@392999149) -> (continuation of Deferred@1610617121 after retry RPC after error@1161964027) -> (continuation of Deferred@1532639517 after retry RPC after error@700645699)))
06:34:04.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.390 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2047337179(state=PENDING, result=null, callback=(continuation of Deferred@753866969 after retry RPC after error@1712304382) -> (continuation of Deferred@2093005353 after retry RPC after error@957648464) -> (continuation of Deferred@1835626619 after retry RPC after error@1987753320) -> (continuation of Deferred@261245052 after retry RPC after error@308879824) -> (continuation of Deferred@883257678 after retry RPC after error@391594823) -> (continuation of Deferred@169551993 after retry RPC after error@1369960267) -> (continuation of Deferred@1310186288 after retry RPC after error@1668801742), errback=(continuation of Deferred@753866969 after retry RPC after error@1712304382) -> (continuation of Deferred@2093005353 after retry RPC after error@957648464) -> (continuation of Deferred@1835626619 after retry RPC after error@1987753320) -> (continuation of Deferred@261245052 after retry RPC after error@308879824) -> (continuation of Deferred@883257678 after retry RPC after error@391594823) -> (continuation of Deferred@169551993 after retry RPC after error@1369960267) -> (continuation of Deferred@1310186288 after retry RPC after error@1668801742)))
06:34:04.391 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2047337179(state=PENDING, result=null, callback=(continuation of Deferred@753866969 after retry RPC after error@1712304382) -> (continuation of Deferred@2093005353 after retry RPC after error@957648464) -> (continuation of Deferred@1835626619 after retry RPC after error@1987753320) -> (continuation of Deferred@261245052 after retry RPC after error@308879824) -> (continuation of Deferred@883257678 after retry RPC after error@391594823) -> (continuation of Deferred@169551993 after retry RPC after error@1369960267) -> (continuation of Deferred@1310186288 after retry RPC after error@1668801742), errback=(continuation of Deferred@753866969 after retry RPC after error@1712304382) -> (continuation of Deferred@2093005353 after retry RPC after error@957648464) -> (continuation of Deferred@1835626619 after retry RPC after error@1987753320) -> (continuation of Deferred@261245052 after retry RPC after error@308879824) -> (continuation of Deferred@883257678 after retry RPC after error@391594823) -> (continuation of Deferred@169551993 after retry RPC after error@1369960267) -> (continuation of Deferred@1310186288 after retry RPC after error@1668801742)))
06:34:04.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.668 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1717707516(state=PENDING, result=null, callback=(continuation of Deferred@803177196 after retry RPC after error@954382741) -> (continuation of Deferred@2076523818 after retry RPC after error@214402412) -> (continuation of Deferred@1608298234 after retry RPC after error@1927252613) -> (continuation of Deferred@547476889 after retry RPC after error@276678660) -> (continuation of Deferred@720113551 after retry RPC after error@610874357) -> (continuation of Deferred@181024764 after retry RPC after error@1955193673) -> (continuation of Deferred@1054018080 after retry RPC after error@1405644948), errback=(continuation of Deferred@803177196 after retry RPC after error@954382741) -> (continuation of Deferred@2076523818 after retry RPC after error@214402412) -> (continuation of Deferred@1608298234 after retry RPC after error@1927252613) -> (continuation of Deferred@547476889 after retry RPC after error@276678660) -> (continuation of Deferred@720113551 after retry RPC after error@610874357) -> (continuation of Deferred@181024764 after retry RPC after error@1955193673) -> (continuation of Deferred@1054018080 after retry RPC after error@1405644948)))
06:34:04.669 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:04.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.772 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.772 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.773 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.773 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.773 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.773 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.773 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.774 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.810 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.830 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.948 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] refreshing cache from master, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [86ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1451403075(state=PENDING, result=null, callback=(continuation of Deferred@218934530 after retry RPC after error@109991270) -> (continuation of Deferred@462289089 after retry RPC after error@1492577356) -> (continuation of Deferred@813317869 after retry RPC after error@1152084193) -> (continuation of Deferred@806673221 after retry RPC after error@1601164679) -> (continuation of Deferred@1048774097 after retry RPC after error@561024640) -> (continuation of Deferred@933803800 after retry RPC after error@373989255) -> (continuation of Deferred@818003218 after retry RPC after error@358625540), errback=(continuation of Deferred@218934530 after retry RPC after error@109991270) -> (continuation of Deferred@462289089 after retry RPC after error@1492577356) -> (continuation of Deferred@813317869 after retry RPC after error@1152084193) -> (continuation of Deferred@806673221 after retry RPC after error@1601164679) -> (continuation of Deferred@1048774097 after retry RPC after error@561024640) -> (continuation of Deferred@933803800 after retry RPC after error@373989255) -> (continuation of Deferred@818003218 after retry RPC after error@358625540)))
06:34:04.949 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:04.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.970 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.970 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:04.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:04.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:04.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:04.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:04.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.011 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.126 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:05.125934 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:05.128 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:05.128321 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:05.129 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:05.129622 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:05.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.191 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1202335868(state=PENDING, result=null, callback=(continuation of Deferred@74001570 after retry RPC after error@875999837) -> (continuation of Deferred@447273018 after retry RPC after error@211231427) -> (continuation of Deferred@1096160207 after retry RPC after error@1081341410) -> (continuation of Deferred@474389726 after retry RPC after error@695346351) -> (continuation of Deferred@1489736229 after retry RPC after error@130592827) -> (continuation of Deferred@792755312 after retry RPC after error@388985451) -> (continuation of Deferred@334598161 after retry RPC after error@1375380284), errback=(continuation of Deferred@74001570 after retry RPC after error@875999837) -> (continuation of Deferred@447273018 after retry RPC after error@211231427) -> (continuation of Deferred@1096160207 after retry RPC after error@1081341410) -> (continuation of Deferred@474389726 after retry RPC after error@695346351) -> (continuation of Deferred@1489736229 after retry RPC after error@130592827) -> (continuation of Deferred@792755312 after retry RPC after error@388985451) -> (continuation of Deferred@334598161 after retry RPC after error@1375380284)))
06:34:05.191 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1202335868(state=PENDING, result=null, callback=(continuation of Deferred@74001570 after retry RPC after error@875999837) -> (continuation of Deferred@447273018 after retry RPC after error@211231427) -> (continuation of Deferred@1096160207 after retry RPC after error@1081341410) -> (continuation of Deferred@474389726 after retry RPC after error@695346351) -> (continuation of Deferred@1489736229 after retry RPC after error@130592827) -> (continuation of Deferred@792755312 after retry RPC after error@388985451) -> (continuation of Deferred@334598161 after retry RPC after error@1375380284), errback=(continuation of Deferred@74001570 after retry RPC after error@875999837) -> (continuation of Deferred@447273018 after retry RPC after error@211231427) -> (continuation of Deferred@1096160207 after retry RPC after error@1081341410) -> (continuation of Deferred@474389726 after retry RPC after error@695346351) -> (continuation of Deferred@1489736229 after retry RPC after error@130592827) -> (continuation of Deferred@792755312 after retry RPC after error@388985451) -> (continuation of Deferred@334598161 after retry RPC after error@1375380284)))
06:34:05.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.228 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.229 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.288 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.288 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.288 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.288 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.289 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.411 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1820162275(state=PENDING, result=null, callback=(continuation of Deferred@1003239265 after retry RPC after error@644622235) -> (continuation of Deferred@394950391 after retry RPC after error@280477317) -> (continuation of Deferred@1251669197 after retry RPC after error@227365626) -> (continuation of Deferred@1232380018 after retry RPC after error@1687841871) -> (continuation of Deferred@2033959053 after retry RPC after error@507922386) -> (continuation of Deferred@1918702424 after retry RPC after error@1158865224), errback=(continuation of Deferred@1003239265 after retry RPC after error@644622235) -> (continuation of Deferred@394950391 after retry RPC after error@280477317) -> (continuation of Deferred@1251669197 after retry RPC after error@227365626) -> (continuation of Deferred@1232380018 after retry RPC after error@1687841871) -> (continuation of Deferred@2033959053 after retry RPC after error@507922386) -> (continuation of Deferred@1918702424 after retry RPC after error@1158865224)))
06:34:05.411 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1820162275(state=PENDING, result=null, callback=(continuation of Deferred@1003239265 after retry RPC after error@644622235) -> (continuation of Deferred@394950391 after retry RPC after error@280477317) -> (continuation of Deferred@1251669197 after retry RPC after error@227365626) -> (continuation of Deferred@1232380018 after retry RPC after error@1687841871) -> (continuation of Deferred@2033959053 after retry RPC after error@507922386) -> (continuation of Deferred@1918702424 after retry RPC after error@1158865224), errback=(continuation of Deferred@1003239265 after retry RPC after error@644622235) -> (continuation of Deferred@394950391 after retry RPC after error@280477317) -> (continuation of Deferred@1251669197 after retry RPC after error@227365626) -> (continuation of Deferred@1232380018 after retry RPC after error@1687841871) -> (continuation of Deferred@2033959053 after retry RPC after error@507922386) -> (continuation of Deferred@1918702424 after retry RPC after error@1158865224)))
06:34:05.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.448 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.448 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.448 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.449 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.610 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@765861469(state=PENDING, result=null, callback=(continuation of Deferred@224788915 after retry RPC after error@111587309) -> (continuation of Deferred@100558796 after retry RPC after error@414701016) -> (continuation of Deferred@635516132 after retry RPC after error@369835634) -> (continuation of Deferred@1623059320 after retry RPC after error@1182989218) -> (continuation of Deferred@1785246082 after retry RPC after error@627042835) -> (continuation of Deferred@716583915 after retry RPC after error@2067305204) -> (continuation of Deferred@1425266864 after retry RPC after error@245804879), errback=(continuation of Deferred@224788915 after retry RPC after error@111587309) -> (continuation of Deferred@100558796 after retry RPC after error@414701016) -> (continuation of Deferred@635516132 after retry RPC after error@369835634) -> (continuation of Deferred@1623059320 after retry RPC after error@1182989218) -> (continuation of Deferred@1785246082 after retry RPC after error@627042835) -> (continuation of Deferred@716583915 after retry RPC after error@2067305204) -> (continuation of Deferred@1425266864 after retry RPC after error@245804879)))
06:34:05.611 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@765861469(state=PENDING, result=null, callback=(continuation of Deferred@224788915 after retry RPC after error@111587309) -> (continuation of Deferred@100558796 after retry RPC after error@414701016) -> (continuation of Deferred@635516132 after retry RPC after error@369835634) -> (continuation of Deferred@1623059320 after retry RPC after error@1182989218) -> (continuation of Deferred@1785246082 after retry RPC after error@627042835) -> (continuation of Deferred@716583915 after retry RPC after error@2067305204) -> (continuation of Deferred@1425266864 after retry RPC after error@245804879), errback=(continuation of Deferred@224788915 after retry RPC after error@111587309) -> (continuation of Deferred@100558796 after retry RPC after error@414701016) -> (continuation of Deferred@635516132 after retry RPC after error@369835634) -> (continuation of Deferred@1623059320 after retry RPC after error@1182989218) -> (continuation of Deferred@1785246082 after retry RPC after error@627042835) -> (continuation of Deferred@716583915 after retry RPC after error@2067305204) -> (continuation of Deferred@1425266864 after retry RPC after error@245804879)))
06:34:05.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.688 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.689 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.790 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@529245013(state=PENDING, result=null, callback=(continuation of Deferred@1098709561 after retry RPC after error@787835406) -> (continuation of Deferred@1727658644 after retry RPC after error@615152661) -> (continuation of Deferred@35776395 after retry RPC after error@125917403) -> (continuation of Deferred@780480485 after retry RPC after error@1505277172) -> (continuation of Deferred@1420303863 after retry RPC after error@1508036665) -> (continuation of Deferred@1579099879 after retry RPC after error@1117529796), errback=(continuation of Deferred@1098709561 after retry RPC after error@787835406) -> (continuation of Deferred@1727658644 after retry RPC after error@615152661) -> (continuation of Deferred@35776395 after retry RPC after error@125917403) -> (continuation of Deferred@780480485 after retry RPC after error@1505277172) -> (continuation of Deferred@1420303863 after retry RPC after error@1508036665) -> (continuation of Deferred@1579099879 after retry RPC after error@1117529796)))
06:34:05.791 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@529245013(state=PENDING, result=null, callback=(continuation of Deferred@1098709561 after retry RPC after error@787835406) -> (continuation of Deferred@1727658644 after retry RPC after error@615152661) -> (continuation of Deferred@35776395 after retry RPC after error@125917403) -> (continuation of Deferred@780480485 after retry RPC after error@1505277172) -> (continuation of Deferred@1420303863 after retry RPC after error@1508036665) -> (continuation of Deferred@1579099879 after retry RPC after error@1117529796), errback=(continuation of Deferred@1098709561 after retry RPC after error@787835406) -> (continuation of Deferred@1727658644 after retry RPC after error@615152661) -> (continuation of Deferred@35776395 after retry RPC after error@125917403) -> (continuation of Deferred@780480485 after retry RPC after error@1505277172) -> (continuation of Deferred@1420303863 after retry RPC after error@1508036665) -> (continuation of Deferred@1579099879 after retry RPC after error@1117529796)))
06:34:05.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.808 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.808 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.890 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.891 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:05.971 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1406401688(state=PENDING, result=null, callback=(continuation of Deferred@1857841932 after retry RPC after error@1155396875) -> (continuation of Deferred@601512947 after retry RPC after error@1864451116) -> (continuation of Deferred@1648928463 after retry RPC after error@1226203680) -> (continuation of Deferred@945995058 after retry RPC after error@1560776246) -> (continuation of Deferred@2061383047 after retry RPC after error@10706846) -> (continuation of Deferred@763754901 after retry RPC after error@1276290093), errback=(continuation of Deferred@1857841932 after retry RPC after error@1155396875) -> (continuation of Deferred@601512947 after retry RPC after error@1864451116) -> (continuation of Deferred@1648928463 after retry RPC after error@1226203680) -> (continuation of Deferred@945995058 after retry RPC after error@1560776246) -> (continuation of Deferred@2061383047 after retry RPC after error@10706846) -> (continuation of Deferred@763754901 after retry RPC after error@1276290093)))
06:34:05.971 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1406401688(state=PENDING, result=null, callback=(continuation of Deferred@1857841932 after retry RPC after error@1155396875) -> (continuation of Deferred@601512947 after retry RPC after error@1864451116) -> (continuation of Deferred@1648928463 after retry RPC after error@1226203680) -> (continuation of Deferred@945995058 after retry RPC after error@1560776246) -> (continuation of Deferred@2061383047 after retry RPC after error@10706846) -> (continuation of Deferred@763754901 after retry RPC after error@1276290093), errback=(continuation of Deferred@1857841932 after retry RPC after error@1155396875) -> (continuation of Deferred@601512947 after retry RPC after error@1864451116) -> (continuation of Deferred@1648928463 after retry RPC after error@1226203680) -> (continuation of Deferred@945995058 after retry RPC after error@1560776246) -> (continuation of Deferred@2061383047 after retry RPC after error@10706846) -> (continuation of Deferred@763754901 after retry RPC after error@1276290093)))
06:34:05.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:05.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:05.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:05.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:05.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.070 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.211 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1871979952(state=PENDING, result=null, callback=(continuation of Deferred@1773729044 after retry RPC after error@1412128426) -> (continuation of Deferred@1098379583 after retry RPC after error@251720964) -> (continuation of Deferred@557756837 after retry RPC after error@804152614) -> (continuation of Deferred@1366202349 after retry RPC after error@1506730001) -> (continuation of Deferred@229228820 after retry RPC after error@1362248402) -> (continuation of Deferred@1959163622 after retry RPC after error@1512492551) -> (continuation of Deferred@423892278 after retry RPC after error@1512300574), errback=(continuation of Deferred@1773729044 after retry RPC after error@1412128426) -> (continuation of Deferred@1098379583 after retry RPC after error@251720964) -> (continuation of Deferred@557756837 after retry RPC after error@804152614) -> (continuation of Deferred@1366202349 after retry RPC after error@1506730001) -> (continuation of Deferred@229228820 after retry RPC after error@1362248402) -> (continuation of Deferred@1959163622 after retry RPC after error@1512492551) -> (continuation of Deferred@423892278 after retry RPC after error@1512300574)))
06:34:06.211 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1871979952(state=PENDING, result=null, callback=(continuation of Deferred@1773729044 after retry RPC after error@1412128426) -> (continuation of Deferred@1098379583 after retry RPC after error@251720964) -> (continuation of Deferred@557756837 after retry RPC after error@804152614) -> (continuation of Deferred@1366202349 after retry RPC after error@1506730001) -> (continuation of Deferred@229228820 after retry RPC after error@1362248402) -> (continuation of Deferred@1959163622 after retry RPC after error@1512492551) -> (continuation of Deferred@423892278 after retry RPC after error@1512300574), errback=(continuation of Deferred@1773729044 after retry RPC after error@1412128426) -> (continuation of Deferred@1098379583 after retry RPC after error@251720964) -> (continuation of Deferred@557756837 after retry RPC after error@804152614) -> (continuation of Deferred@1366202349 after retry RPC after error@1506730001) -> (continuation of Deferred@229228820 after retry RPC after error@1362248402) -> (continuation of Deferred@1959163622 after retry RPC after error@1512492551) -> (continuation of Deferred@423892278 after retry RPC after error@1512300574)))
06:34:06.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.250 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.310 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.489 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1951598908(state=PENDING, result=null, callback=(continuation of Deferred@1692759304 after retry RPC after error@505666319) -> (continuation of Deferred@436101010 after retry RPC after error@1971892088) -> (continuation of Deferred@973789684 after retry RPC after error@980522521) -> (continuation of Deferred@1368939530 after retry RPC after error@1633409582) -> (continuation of Deferred@225606280 after retry RPC after error@990241889) -> (continuation of Deferred@678332167 after retry RPC after error@400274062) -> (continuation of Deferred@165887175 after retry RPC after error@491911030), errback=(continuation of Deferred@1692759304 after retry RPC after error@505666319) -> (continuation of Deferred@436101010 after retry RPC after error@1971892088) -> (continuation of Deferred@973789684 after retry RPC after error@980522521) -> (continuation of Deferred@1368939530 after retry RPC after error@1633409582) -> (continuation of Deferred@225606280 after retry RPC after error@990241889) -> (continuation of Deferred@678332167 after retry RPC after error@400274062) -> (continuation of Deferred@165887175 after retry RPC after error@491911030)))
06:34:06.489 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:06.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.530 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.530 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.530 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.531 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.548 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.549 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.691 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@702642072(state=PENDING, result=null, callback=(continuation of Deferred@1703724211 after retry RPC after error@1282778560) -> (continuation of Deferred@978160755 after retry RPC after error@414343445) -> (continuation of Deferred@462981100 after retry RPC after error@2049080058) -> (continuation of Deferred@744120281 after retry RPC after error@669727118) -> (continuation of Deferred@2022927736 after retry RPC after error@294599994) -> (continuation of Deferred@1867848773 after retry RPC after error@294416232) -> (continuation of Deferred@1922513036 after retry RPC after error@437890313), errback=(continuation of Deferred@1703724211 after retry RPC after error@1282778560) -> (continuation of Deferred@978160755 after retry RPC after error@414343445) -> (continuation of Deferred@462981100 after retry RPC after error@2049080058) -> (continuation of Deferred@744120281 after retry RPC after error@669727118) -> (continuation of Deferred@2022927736 after retry RPC after error@294599994) -> (continuation of Deferred@1867848773 after retry RPC after error@294416232) -> (continuation of Deferred@1922513036 after retry RPC after error@437890313)))
06:34:06.691 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@702642072(state=PENDING, result=null, callback=(continuation of Deferred@1703724211 after retry RPC after error@1282778560) -> (continuation of Deferred@978160755 after retry RPC after error@414343445) -> (continuation of Deferred@462981100 after retry RPC after error@2049080058) -> (continuation of Deferred@744120281 after retry RPC after error@669727118) -> (continuation of Deferred@2022927736 after retry RPC after error@294599994) -> (continuation of Deferred@1867848773 after retry RPC after error@294416232) -> (continuation of Deferred@1922513036 after retry RPC after error@437890313), errback=(continuation of Deferred@1703724211 after retry RPC after error@1282778560) -> (continuation of Deferred@978160755 after retry RPC after error@414343445) -> (continuation of Deferred@462981100 after retry RPC after error@2049080058) -> (continuation of Deferred@744120281 after retry RPC after error@669727118) -> (continuation of Deferred@2022927736 after retry RPC after error@294599994) -> (continuation of Deferred@1867848773 after retry RPC after error@294416232) -> (continuation of Deferred@1922513036 after retry RPC after error@437890313)))
06:34:06.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.830 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.830 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.890 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.890 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.891 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.891 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1741013482(state=PENDING, result=null, callback=(continuation of Deferred@262790156 after retry RPC after error@2049723027) -> (continuation of Deferred@1836832699 after retry RPC after error@620269935) -> (continuation of Deferred@1710256192 after retry RPC after error@2106640063) -> (continuation of Deferred@1315246265 after retry RPC after error@779523059) -> (continuation of Deferred@1214415885 after retry RPC after error@867960448) -> (continuation of Deferred@1438263446 after retry RPC after error@1330233561), errback=(continuation of Deferred@262790156 after retry RPC after error@2049723027) -> (continuation of Deferred@1836832699 after retry RPC after error@620269935) -> (continuation of Deferred@1710256192 after retry RPC after error@2106640063) -> (continuation of Deferred@1315246265 after retry RPC after error@779523059) -> (continuation of Deferred@1214415885 after retry RPC after error@867960448) -> (continuation of Deferred@1438263446 after retry RPC after error@1330233561)))
06:34:06.891 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1741013482(state=PENDING, result=null, callback=(continuation of Deferred@262790156 after retry RPC after error@2049723027) -> (continuation of Deferred@1836832699 after retry RPC after error@620269935) -> (continuation of Deferred@1710256192 after retry RPC after error@2106640063) -> (continuation of Deferred@1315246265 after retry RPC after error@779523059) -> (continuation of Deferred@1214415885 after retry RPC after error@867960448) -> (continuation of Deferred@1438263446 after retry RPC after error@1330233561), errback=(continuation of Deferred@262790156 after retry RPC after error@2049723027) -> (continuation of Deferred@1836832699 after retry RPC after error@620269935) -> (continuation of Deferred@1710256192 after retry RPC after error@2106640063) -> (continuation of Deferred@1315246265 after retry RPC after error@779523059) -> (continuation of Deferred@1214415885 after retry RPC after error@867960448) -> (continuation of Deferred@1438263446 after retry RPC after error@1330233561)))
06:34:06.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.968 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:06.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:06.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:06.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:06.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:06.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.151 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1792613998(state=PENDING, result=null, callback=(continuation of Deferred@1083075723 after retry RPC after error@457237576) -> (continuation of Deferred@1360145317 after retry RPC after error@1939502471) -> (continuation of Deferred@1086563659 after retry RPC after error@1555921855) -> (continuation of Deferred@481438862 after retry RPC after error@1009009874) -> (continuation of Deferred@1628461024 after retry RPC after error@1447536894) -> (continuation of Deferred@1536964836 after retry RPC after error@1166079377) -> (continuation of Deferred@642520470 after retry RPC after error@1302813177), errback=(continuation of Deferred@1083075723 after retry RPC after error@457237576) -> (continuation of Deferred@1360145317 after retry RPC after error@1939502471) -> (continuation of Deferred@1086563659 after retry RPC after error@1555921855) -> (continuation of Deferred@481438862 after retry RPC after error@1009009874) -> (continuation of Deferred@1628461024 after retry RPC after error@1447536894) -> (continuation of Deferred@1536964836 after retry RPC after error@1166079377) -> (continuation of Deferred@642520470 after retry RPC after error@1302813177)))
06:34:07.151 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1792613998(state=PENDING, result=null, callback=(continuation of Deferred@1083075723 after retry RPC after error@457237576) -> (continuation of Deferred@1360145317 after retry RPC after error@1939502471) -> (continuation of Deferred@1086563659 after retry RPC after error@1555921855) -> (continuation of Deferred@481438862 after retry RPC after error@1009009874) -> (continuation of Deferred@1628461024 after retry RPC after error@1447536894) -> (continuation of Deferred@1536964836 after retry RPC after error@1166079377) -> (continuation of Deferred@642520470 after retry RPC after error@1302813177), errback=(continuation of Deferred@1083075723 after retry RPC after error@457237576) -> (continuation of Deferred@1360145317 after retry RPC after error@1939502471) -> (continuation of Deferred@1086563659 after retry RPC after error@1555921855) -> (continuation of Deferred@481438862 after retry RPC after error@1009009874) -> (continuation of Deferred@1628461024 after retry RPC after error@1447536894) -> (continuation of Deferred@1536964836 after retry RPC after error@1166079377) -> (continuation of Deferred@642520470 after retry RPC after error@1302813177)))
06:34:07.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.190 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.190 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.190 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.310 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.311 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:07.322045 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:07.322 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:07.322259 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:07.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.410 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@427469137(state=PENDING, result=null, callback=(continuation of Deferred@1351026327 after retry RPC after error@1817238265) -> (continuation of Deferred@1947555368 after retry RPC after error@954812890) -> (continuation of Deferred@1345475566 after retry RPC after error@987749379) -> (continuation of Deferred@1688566118 after retry RPC after error@384846769) -> (continuation of Deferred@1411372962 after retry RPC after error@275003110) -> (continuation of Deferred@1598358469 after retry RPC after error@1274836575) -> (continuation of Deferred@1634994757 after retry RPC after error@1343669131), errback=(continuation of Deferred@1351026327 after retry RPC after error@1817238265) -> (continuation of Deferred@1947555368 after retry RPC after error@954812890) -> (continuation of Deferred@1345475566 after retry RPC after error@987749379) -> (continuation of Deferred@1688566118 after retry RPC after error@384846769) -> (continuation of Deferred@1411372962 after retry RPC after error@275003110) -> (continuation of Deferred@1598358469 after retry RPC after error@1274836575) -> (continuation of Deferred@1634994757 after retry RPC after error@1343669131)))
06:34:07.411 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@427469137(state=PENDING, result=null, callback=(continuation of Deferred@1351026327 after retry RPC after error@1817238265) -> (continuation of Deferred@1947555368 after retry RPC after error@954812890) -> (continuation of Deferred@1345475566 after retry RPC after error@987749379) -> (continuation of Deferred@1688566118 after retry RPC after error@384846769) -> (continuation of Deferred@1411372962 after retry RPC after error@275003110) -> (continuation of Deferred@1598358469 after retry RPC after error@1274836575) -> (continuation of Deferred@1634994757 after retry RPC after error@1343669131), errback=(continuation of Deferred@1351026327 after retry RPC after error@1817238265) -> (continuation of Deferred@1947555368 after retry RPC after error@954812890) -> (continuation of Deferred@1345475566 after retry RPC after error@987749379) -> (continuation of Deferred@1688566118 after retry RPC after error@384846769) -> (continuation of Deferred@1411372962 after retry RPC after error@275003110) -> (continuation of Deferred@1598358469 after retry RPC after error@1274836575) -> (continuation of Deferred@1634994757 after retry RPC after error@1343669131)))
06:34:07.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.430 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.430 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.491 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.568 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.568 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.568 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.568 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.569 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.630 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.631 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [139ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1170696501(state=PENDING, result=null, callback=(continuation of Deferred@1818587748 after retry RPC after error@1885575126) -> (continuation of Deferred@1520222904 after retry RPC after error@611091999) -> (continuation of Deferred@1667049124 after retry RPC after error@267173950) -> (continuation of Deferred@869882489 after retry RPC after error@1942322787) -> (continuation of Deferred@2040995513 after retry RPC after error@279170179) -> (continuation of Deferred@1766868604 after retry RPC after error@826153811) -> (continuation of Deferred@1882036507 after retry RPC after error@2120516681), errback=(continuation of Deferred@1818587748 after retry RPC after error@1885575126) -> (continuation of Deferred@1520222904 after retry RPC after error@611091999) -> (continuation of Deferred@1667049124 after retry RPC after error@267173950) -> (continuation of Deferred@869882489 after retry RPC after error@1942322787) -> (continuation of Deferred@2040995513 after retry RPC after error@279170179) -> (continuation of Deferred@1766868604 after retry RPC after error@826153811) -> (continuation of Deferred@1882036507 after retry RPC after error@2120516681)))
06:34:07.632 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [139ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1170696501(state=PENDING, result=null, callback=(continuation of Deferred@1818587748 after retry RPC after error@1885575126) -> (continuation of Deferred@1520222904 after retry RPC after error@611091999) -> (continuation of Deferred@1667049124 after retry RPC after error@267173950) -> (continuation of Deferred@869882489 after retry RPC after error@1942322787) -> (continuation of Deferred@2040995513 after retry RPC after error@279170179) -> (continuation of Deferred@1766868604 after retry RPC after error@826153811) -> (continuation of Deferred@1882036507 after retry RPC after error@2120516681), errback=(continuation of Deferred@1818587748 after retry RPC after error@1885575126) -> (continuation of Deferred@1520222904 after retry RPC after error@611091999) -> (continuation of Deferred@1667049124 after retry RPC after error@267173950) -> (continuation of Deferred@869882489 after retry RPC after error@1942322787) -> (continuation of Deferred@2040995513 after retry RPC after error@279170179) -> (continuation of Deferred@1766868604 after retry RPC after error@826153811) -> (continuation of Deferred@1882036507 after retry RPC after error@2120516681)))
06:34:07.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@511384718(state=PENDING, result=null, callback=(continuation of Deferred@1736514400 after retry RPC after error@933620606) -> (continuation of Deferred@499982416 after retry RPC after error@1536479243) -> (continuation of Deferred@874613250 after retry RPC after error@751266275) -> (continuation of Deferred@1484522379 after retry RPC after error@622171143) -> (continuation of Deferred@492600125 after retry RPC after error@180584708) -> (continuation of Deferred@840678284 after retry RPC after error@992457828), errback=(continuation of Deferred@1736514400 after retry RPC after error@933620606) -> (continuation of Deferred@499982416 after retry RPC after error@1536479243) -> (continuation of Deferred@874613250 after retry RPC after error@751266275) -> (continuation of Deferred@1484522379 after retry RPC after error@622171143) -> (continuation of Deferred@492600125 after retry RPC after error@180584708) -> (continuation of Deferred@840678284 after retry RPC after error@992457828)))
06:34:07.811 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@511384718(state=PENDING, result=null, callback=(continuation of Deferred@1736514400 after retry RPC after error@933620606) -> (continuation of Deferred@499982416 after retry RPC after error@1536479243) -> (continuation of Deferred@874613250 after retry RPC after error@751266275) -> (continuation of Deferred@1484522379 after retry RPC after error@622171143) -> (continuation of Deferred@492600125 after retry RPC after error@180584708) -> (continuation of Deferred@840678284 after retry RPC after error@992457828), errback=(continuation of Deferred@1736514400 after retry RPC after error@933620606) -> (continuation of Deferred@499982416 after retry RPC after error@1536479243) -> (continuation of Deferred@874613250 after retry RPC after error@751266275) -> (continuation of Deferred@1484522379 after retry RPC after error@622171143) -> (continuation of Deferred@492600125 after retry RPC after error@180584708) -> (continuation of Deferred@840678284 after retry RPC after error@992457828)))
06:34:07.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.950 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.950 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.951 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.951 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:07.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:07.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:07.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:07.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:07.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1839734022(state=PENDING, result=null, callback=(continuation of Deferred@1323151620 after retry RPC after error@845077473) -> (continuation of Deferred@1304038864 after retry RPC after error@1624729932) -> (continuation of Deferred@1134666179 after retry RPC after error@124058501) -> (continuation of Deferred@439198260 after retry RPC after error@1097540916) -> (continuation of Deferred@260717411 after retry RPC after error@789729732) -> (continuation of Deferred@1490643472 after retry RPC after error@1082373818) -> (continuation of Deferred@512548074 after retry RPC after error@598549134) -> (continuation of Deferred@808928513 after retry RPC after error@1843139547), errback=(continuation of Deferred@1323151620 after retry RPC after error@845077473) -> (continuation of Deferred@1304038864 after retry RPC after error@1624729932) -> (continuation of Deferred@1134666179 after retry RPC after error@124058501) -> (continuation of Deferred@439198260 after retry RPC after error@1097540916) -> (continuation of Deferred@260717411 after retry RPC after error@789729732) -> (continuation of Deferred@1490643472 after retry RPC after error@1082373818) -> (continuation of Deferred@512548074 after retry RPC after error@598549134) -> (continuation of Deferred@808928513 after retry RPC after error@1843139547)))
06:34:08.072 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] refreshing cache from master, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1839734022(state=PENDING, result=null, callback=(continuation of Deferred@1323151620 after retry RPC after error@845077473) -> (continuation of Deferred@1304038864 after retry RPC after error@1624729932) -> (continuation of Deferred@1134666179 after retry RPC after error@124058501) -> (continuation of Deferred@439198260 after retry RPC after error@1097540916) -> (continuation of Deferred@260717411 after retry RPC after error@789729732) -> (continuation of Deferred@1490643472 after retry RPC after error@1082373818) -> (continuation of Deferred@512548074 after retry RPC after error@598549134) -> (continuation of Deferred@808928513 after retry RPC after error@1843139547), errback=(continuation of Deferred@1323151620 after retry RPC after error@845077473) -> (continuation of Deferred@1304038864 after retry RPC after error@1624729932) -> (continuation of Deferred@1134666179 after retry RPC after error@124058501) -> (continuation of Deferred@439198260 after retry RPC after error@1097540916) -> (continuation of Deferred@260717411 after retry RPC after error@789729732) -> (continuation of Deferred@1490643472 after retry RPC after error@1082373818) -> (continuation of Deferred@512548074 after retry RPC after error@598549134) -> (continuation of Deferred@808928513 after retry RPC after error@1843139547)))
06:34:08.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.088 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.088 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.088 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.089 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.129 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.150 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1754920861(state=PENDING, result=null, callback=(continuation of Deferred@1855373651 after retry RPC after error@2006489293) -> (continuation of Deferred@173066594 after retry RPC after error@768799402) -> (continuation of Deferred@1972773402 after retry RPC after error@1190888085) -> (continuation of Deferred@1248610545 after retry RPC after error@1880208564) -> (continuation of Deferred@877911646 after retry RPC after error@774249835) -> (continuation of Deferred@111535006 after retry RPC after error@2070692842), errback=(continuation of Deferred@1855373651 after retry RPC after error@2006489293) -> (continuation of Deferred@173066594 after retry RPC after error@768799402) -> (continuation of Deferred@1972773402 after retry RPC after error@1190888085) -> (continuation of Deferred@1248610545 after retry RPC after error@1880208564) -> (continuation of Deferred@877911646 after retry RPC after error@774249835) -> (continuation of Deferred@111535006 after retry RPC after error@2070692842)))
06:34:08.271 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1754920861(state=PENDING, result=null, callback=(continuation of Deferred@1855373651 after retry RPC after error@2006489293) -> (continuation of Deferred@173066594 after retry RPC after error@768799402) -> (continuation of Deferred@1972773402 after retry RPC after error@1190888085) -> (continuation of Deferred@1248610545 after retry RPC after error@1880208564) -> (continuation of Deferred@877911646 after retry RPC after error@774249835) -> (continuation of Deferred@111535006 after retry RPC after error@2070692842), errback=(continuation of Deferred@1855373651 after retry RPC after error@2006489293) -> (continuation of Deferred@173066594 after retry RPC after error@768799402) -> (continuation of Deferred@1972773402 after retry RPC after error@1190888085) -> (continuation of Deferred@1248610545 after retry RPC after error@1880208564) -> (continuation of Deferred@877911646 after retry RPC after error@774249835) -> (continuation of Deferred@111535006 after retry RPC after error@2070692842)))
06:34:08.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.430 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2077209528(state=PENDING, result=null, callback=(continuation of Deferred@2062968369 after retry RPC after error@923115634) -> (continuation of Deferred@101683318 after retry RPC after error@14393724) -> (continuation of Deferred@1238730936 after retry RPC after error@1405724608) -> (continuation of Deferred@700387823 after retry RPC after error@1957856741) -> (continuation of Deferred@289883011 after retry RPC after error@254673356) -> (continuation of Deferred@1590647215 after retry RPC after error@1043493438), errback=(continuation of Deferred@2062968369 after retry RPC after error@923115634) -> (continuation of Deferred@101683318 after retry RPC after error@14393724) -> (continuation of Deferred@1238730936 after retry RPC after error@1405724608) -> (continuation of Deferred@700387823 after retry RPC after error@1957856741) -> (continuation of Deferred@289883011 after retry RPC after error@254673356) -> (continuation of Deferred@1590647215 after retry RPC after error@1043493438)))
06:34:08.431 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=141), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2077209528(state=PENDING, result=null, callback=(continuation of Deferred@2062968369 after retry RPC after error@923115634) -> (continuation of Deferred@101683318 after retry RPC after error@14393724) -> (continuation of Deferred@1238730936 after retry RPC after error@1405724608) -> (continuation of Deferred@700387823 after retry RPC after error@1957856741) -> (continuation of Deferred@289883011 after retry RPC after error@254673356) -> (continuation of Deferred@1590647215 after retry RPC after error@1043493438), errback=(continuation of Deferred@2062968369 after retry RPC after error@923115634) -> (continuation of Deferred@101683318 after retry RPC after error@14393724) -> (continuation of Deferred@1238730936 after retry RPC after error@1405724608) -> (continuation of Deferred@700387823 after retry RPC after error@1957856741) -> (continuation of Deferred@289883011 after retry RPC after error@254673356) -> (continuation of Deferred@1590647215 after retry RPC after error@1043493438)))
06:34:08.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.548 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.548 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.651 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1627450270(state=PENDING, result=null, callback=(continuation of Deferred@1742995364 after retry RPC after error@1507893464) -> (continuation of Deferred@1843692770 after retry RPC after error@190762715) -> (continuation of Deferred@1941446347 after retry RPC after error@797087236) -> (continuation of Deferred@1711854722 after retry RPC after error@459142798) -> (continuation of Deferred@681635176 after retry RPC after error@321481934) -> (continuation of Deferred@858068717 after retry RPC after error@1203273601) -> (continuation of Deferred@1750864307 after retry RPC after error@818630506) -> (continuation of Deferred@725792224 after retry RPC after error@797086590), errback=(continuation of Deferred@1742995364 after retry RPC after error@1507893464) -> (continuation of Deferred@1843692770 after retry RPC after error@190762715) -> (continuation of Deferred@1941446347 after retry RPC after error@797087236) -> (continuation of Deferred@1711854722 after retry RPC after error@459142798) -> (continuation of Deferred@681635176 after retry RPC after error@321481934) -> (continuation of Deferred@858068717 after retry RPC after error@1203273601) -> (continuation of Deferred@1750864307 after retry RPC after error@818630506) -> (continuation of Deferred@725792224 after retry RPC after error@797086590)))
06:34:08.651 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1627450270(state=PENDING, result=null, callback=(continuation of Deferred@1742995364 after retry RPC after error@1507893464) -> (continuation of Deferred@1843692770 after retry RPC after error@190762715) -> (continuation of Deferred@1941446347 after retry RPC after error@797087236) -> (continuation of Deferred@1711854722 after retry RPC after error@459142798) -> (continuation of Deferred@681635176 after retry RPC after error@321481934) -> (continuation of Deferred@858068717 after retry RPC after error@1203273601) -> (continuation of Deferred@1750864307 after retry RPC after error@818630506) -> (continuation of Deferred@725792224 after retry RPC after error@797086590), errback=(continuation of Deferred@1742995364 after retry RPC after error@1507893464) -> (continuation of Deferred@1843692770 after retry RPC after error@190762715) -> (continuation of Deferred@1941446347 after retry RPC after error@797087236) -> (continuation of Deferred@1711854722 after retry RPC after error@459142798) -> (continuation of Deferred@681635176 after retry RPC after error@321481934) -> (continuation of Deferred@858068717 after retry RPC after error@1203273601) -> (continuation of Deferred@1750864307 after retry RPC after error@818630506) -> (continuation of Deferred@725792224 after retry RPC after error@797086590)))
06:34:08.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.750 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.870 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.928 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@149782733(state=PENDING, result=null, callback=(continuation of Deferred@1041723584 after retry RPC after error@1517398755) -> (continuation of Deferred@1959959835 after retry RPC after error@274567332) -> (continuation of Deferred@357135135 after retry RPC after error@1111029223) -> (continuation of Deferred@585274110 after retry RPC after error@1590099638) -> (continuation of Deferred@13221882 after retry RPC after error@89908681) -> (continuation of Deferred@857273086 after retry RPC after error@1522698597) -> (continuation of Deferred@398052264 after retry RPC after error@1293287739) -> (continuation of Deferred@963478885 after retry RPC after error@1999288972), errback=(continuation of Deferred@1041723584 after retry RPC after error@1517398755) -> (continuation of Deferred@1959959835 after retry RPC after error@274567332) -> (continuation of Deferred@357135135 after retry RPC after error@1111029223) -> (continuation of Deferred@585274110 after retry RPC after error@1590099638) -> (continuation of Deferred@13221882 after retry RPC after error@89908681) -> (continuation of Deferred@857273086 after retry RPC after error@1522698597) -> (continuation of Deferred@398052264 after retry RPC after error@1293287739) -> (continuation of Deferred@963478885 after retry RPC after error@1999288972)))
06:34:08.929 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:08.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:08.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:08.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:08.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:08.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:08.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.171 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@193772534(state=PENDING, result=null, callback=(continuation of Deferred@340778448 after retry RPC after error@788948119) -> (continuation of Deferred@1970011520 after retry RPC after error@467053520) -> (continuation of Deferred@1906132007 after retry RPC after error@1831056479) -> (continuation of Deferred@1200542598 after retry RPC after error@1764255884) -> (continuation of Deferred@1913632242 after retry RPC after error@41842336) -> (continuation of Deferred@870160266 after retry RPC after error@421262709) -> (continuation of Deferred@380920705 after retry RPC after error@1729023105), errback=(continuation of Deferred@340778448 after retry RPC after error@788948119) -> (continuation of Deferred@1970011520 after retry RPC after error@467053520) -> (continuation of Deferred@1906132007 after retry RPC after error@1831056479) -> (continuation of Deferred@1200542598 after retry RPC after error@1764255884) -> (continuation of Deferred@1913632242 after retry RPC after error@41842336) -> (continuation of Deferred@870160266 after retry RPC after error@421262709) -> (continuation of Deferred@380920705 after retry RPC after error@1729023105)))
06:34:09.172 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@193772534(state=PENDING, result=null, callback=(continuation of Deferred@340778448 after retry RPC after error@788948119) -> (continuation of Deferred@1970011520 after retry RPC after error@467053520) -> (continuation of Deferred@1906132007 after retry RPC after error@1831056479) -> (continuation of Deferred@1200542598 after retry RPC after error@1764255884) -> (continuation of Deferred@1913632242 after retry RPC after error@41842336) -> (continuation of Deferred@870160266 after retry RPC after error@421262709) -> (continuation of Deferred@380920705 after retry RPC after error@1729023105), errback=(continuation of Deferred@340778448 after retry RPC after error@788948119) -> (continuation of Deferred@1970011520 after retry RPC after error@467053520) -> (continuation of Deferred@1906132007 after retry RPC after error@1831056479) -> (continuation of Deferred@1200542598 after retry RPC after error@1764255884) -> (continuation of Deferred@1913632242 after retry RPC after error@41842336) -> (continuation of Deferred@870160266 after retry RPC after error@421262709) -> (continuation of Deferred@380920705 after retry RPC after error@1729023105)))
06:34:09.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.350 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.350 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1018080606(state=PENDING, result=null, callback=(continuation of Deferred@1618664637 after retry RPC after error@808446630) -> (continuation of Deferred@30844755 after retry RPC after error@1222323818) -> (continuation of Deferred@2121436408 after retry RPC after error@826745674) -> (continuation of Deferred@94369294 after retry RPC after error@346431657) -> (continuation of Deferred@2119789449 after retry RPC after error@1980516987) -> (continuation of Deferred@215608626 after retry RPC after error@702494568), errback=(continuation of Deferred@1618664637 after retry RPC after error@808446630) -> (continuation of Deferred@30844755 after retry RPC after error@1222323818) -> (continuation of Deferred@2121436408 after retry RPC after error@826745674) -> (continuation of Deferred@94369294 after retry RPC after error@346431657) -> (continuation of Deferred@2119789449 after retry RPC after error@1980516987) -> (continuation of Deferred@215608626 after retry RPC after error@702494568)))
06:34:09.351 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1018080606(state=PENDING, result=null, callback=(continuation of Deferred@1618664637 after retry RPC after error@808446630) -> (continuation of Deferred@30844755 after retry RPC after error@1222323818) -> (continuation of Deferred@2121436408 after retry RPC after error@826745674) -> (continuation of Deferred@94369294 after retry RPC after error@346431657) -> (continuation of Deferred@2119789449 after retry RPC after error@1980516987) -> (continuation of Deferred@215608626 after retry RPC after error@702494568), errback=(continuation of Deferred@1618664637 after retry RPC after error@808446630) -> (continuation of Deferred@30844755 after retry RPC after error@1222323818) -> (continuation of Deferred@2121436408 after retry RPC after error@826745674) -> (continuation of Deferred@94369294 after retry RPC after error@346431657) -> (continuation of Deferred@2119789449 after retry RPC after error@1980516987) -> (continuation of Deferred@215608626 after retry RPC after error@702494568)))
06:34:09.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.470 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.470 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.471 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.471 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.571 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:09.571550 25690 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:34:09.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:09.571779 25690 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:34:09.572 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:09.572415 25690 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:34:09.576 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:09.576422 25333 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:34:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:09.577878 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:34:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:09.578070 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:34:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:09.578176 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:34:09.578 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:09.578408 25690 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:34:09.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.610 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@184118041(state=PENDING, result=null, callback=(continuation of Deferred@1336622383 after retry RPC after error@1828365967) -> (continuation of Deferred@1389079363 after retry RPC after error@321631136) -> (continuation of Deferred@92251823 after retry RPC after error@1007199825) -> (continuation of Deferred@857113376 after retry RPC after error@2104397905) -> (continuation of Deferred@89918349 after retry RPC after error@2082414425) -> (continuation of Deferred@2065144953 after retry RPC after error@1928488786) -> (continuation of Deferred@387386082 after retry RPC after error@326511246), errback=(continuation of Deferred@1336622383 after retry RPC after error@1828365967) -> (continuation of Deferred@1389079363 after retry RPC after error@321631136) -> (continuation of Deferred@92251823 after retry RPC after error@1007199825) -> (continuation of Deferred@857113376 after retry RPC after error@2104397905) -> (continuation of Deferred@89918349 after retry RPC after error@2082414425) -> (continuation of Deferred@2065144953 after retry RPC after error@1928488786) -> (continuation of Deferred@387386082 after retry RPC after error@326511246)))
06:34:09.611 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@184118041(state=PENDING, result=null, callback=(continuation of Deferred@1336622383 after retry RPC after error@1828365967) -> (continuation of Deferred@1389079363 after retry RPC after error@321631136) -> (continuation of Deferred@92251823 after retry RPC after error@1007199825) -> (continuation of Deferred@857113376 after retry RPC after error@2104397905) -> (continuation of Deferred@89918349 after retry RPC after error@2082414425) -> (continuation of Deferred@2065144953 after retry RPC after error@1928488786) -> (continuation of Deferred@387386082 after retry RPC after error@326511246), errback=(continuation of Deferred@1336622383 after retry RPC after error@1828365967) -> (continuation of Deferred@1389079363 after retry RPC after error@321631136) -> (continuation of Deferred@92251823 after retry RPC after error@1007199825) -> (continuation of Deferred@857113376 after retry RPC after error@2104397905) -> (continuation of Deferred@89918349 after retry RPC after error@2082414425) -> (continuation of Deferred@2065144953 after retry RPC after error@1928488786) -> (continuation of Deferred@387386082 after retry RPC after error@326511246)))
06:34:09.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.690 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.729 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.850 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.850 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.852 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1370688548(state=PENDING, result=null, callback=(continuation of Deferred@1846495536 after retry RPC after error@553459491) -> (continuation of Deferred@1775274776 after retry RPC after error@550205482) -> (continuation of Deferred@882500554 after retry RPC after error@2086008445) -> (continuation of Deferred@288373412 after retry RPC after error@1791036032) -> (continuation of Deferred@289223585 after retry RPC after error@1107872229) -> (continuation of Deferred@1392324122 after retry RPC after error@806005711) -> (continuation of Deferred@1213343316 after retry RPC after error@1284606298), errback=(continuation of Deferred@1846495536 after retry RPC after error@553459491) -> (continuation of Deferred@1775274776 after retry RPC after error@550205482) -> (continuation of Deferred@882500554 after retry RPC after error@2086008445) -> (continuation of Deferred@288373412 after retry RPC after error@1791036032) -> (continuation of Deferred@289223585 after retry RPC after error@1107872229) -> (continuation of Deferred@1392324122 after retry RPC after error@806005711) -> (continuation of Deferred@1213343316 after retry RPC after error@1284606298)))
06:34:09.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=223), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1370688548(state=PENDING, result=null, callback=(continuation of Deferred@1846495536 after retry RPC after error@553459491) -> (continuation of Deferred@1775274776 after retry RPC after error@550205482) -> (continuation of Deferred@882500554 after retry RPC after error@2086008445) -> (continuation of Deferred@288373412 after retry RPC after error@1791036032) -> (continuation of Deferred@289223585 after retry RPC after error@1107872229) -> (continuation of Deferred@1392324122 after retry RPC after error@806005711) -> (continuation of Deferred@1213343316 after retry RPC after error@1284606298), errback=(continuation of Deferred@1846495536 after retry RPC after error@553459491) -> (continuation of Deferred@1775274776 after retry RPC after error@550205482) -> (continuation of Deferred@882500554 after retry RPC after error@2086008445) -> (continuation of Deferred@288373412 after retry RPC after error@1791036032) -> (continuation of Deferred@289223585 after retry RPC after error@1107872229) -> (continuation of Deferred@1392324122 after retry RPC after error@806005711) -> (continuation of Deferred@1213343316 after retry RPC after error@1284606298)))
06:34:09.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.870 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.870 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:09.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:09.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:09.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.968 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:09.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:09.969 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.031 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1730081589(state=PENDING, result=null, callback=(continuation of Deferred@1165549613 after retry RPC after error@1564376676) -> (continuation of Deferred@612581505 after retry RPC after error@873764991) -> (continuation of Deferred@921661272 after retry RPC after error@403499711) -> (continuation of Deferred@649410167 after retry RPC after error@921867710) -> (continuation of Deferred@2074782103 after retry RPC after error@67960143) -> (continuation of Deferred@184907063 after retry RPC after error@704138892), errback=(continuation of Deferred@1165549613 after retry RPC after error@1564376676) -> (continuation of Deferred@612581505 after retry RPC after error@873764991) -> (continuation of Deferred@921661272 after retry RPC after error@403499711) -> (continuation of Deferred@649410167 after retry RPC after error@921867710) -> (continuation of Deferred@2074782103 after retry RPC after error@67960143) -> (continuation of Deferred@184907063 after retry RPC after error@704138892)))
06:34:10.031 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1730081589(state=PENDING, result=null, callback=(continuation of Deferred@1165549613 after retry RPC after error@1564376676) -> (continuation of Deferred@612581505 after retry RPC after error@873764991) -> (continuation of Deferred@921661272 after retry RPC after error@403499711) -> (continuation of Deferred@649410167 after retry RPC after error@921867710) -> (continuation of Deferred@2074782103 after retry RPC after error@67960143) -> (continuation of Deferred@184907063 after retry RPC after error@704138892), errback=(continuation of Deferred@1165549613 after retry RPC after error@1564376676) -> (continuation of Deferred@612581505 after retry RPC after error@873764991) -> (continuation of Deferred@921661272 after retry RPC after error@403499711) -> (continuation of Deferred@649410167 after retry RPC after error@921867710) -> (continuation of Deferred@2074782103 after retry RPC after error@67960143) -> (continuation of Deferred@184907063 after retry RPC after error@704138892)))
06:34:10.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.068 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.068 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.152 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:10.151919 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:34:10.156 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:10.156389 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:34:10.157 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:10.157727 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:10.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.270 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1913266424(state=PENDING, result=null, callback=(continuation of Deferred@795245636 after retry RPC after error@1904707019) -> (continuation of Deferred@1750162894 after retry RPC after error@1275047388) -> (continuation of Deferred@1474788539 after retry RPC after error@436408913) -> (continuation of Deferred@2027852786 after retry RPC after error@1324961912) -> (continuation of Deferred@571998136 after retry RPC after error@537190449) -> (continuation of Deferred@743024320 after retry RPC after error@683100577) -> (continuation of Deferred@893421872 after retry RPC after error@843248321), errback=(continuation of Deferred@795245636 after retry RPC after error@1904707019) -> (continuation of Deferred@1750162894 after retry RPC after error@1275047388) -> (continuation of Deferred@1474788539 after retry RPC after error@436408913) -> (continuation of Deferred@2027852786 after retry RPC after error@1324961912) -> (continuation of Deferred@571998136 after retry RPC after error@537190449) -> (continuation of Deferred@743024320 after retry RPC after error@683100577) -> (continuation of Deferred@893421872 after retry RPC after error@843248321)))
06:34:10.271 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1913266424(state=PENDING, result=null, callback=(continuation of Deferred@795245636 after retry RPC after error@1904707019) -> (continuation of Deferred@1750162894 after retry RPC after error@1275047388) -> (continuation of Deferred@1474788539 after retry RPC after error@436408913) -> (continuation of Deferred@2027852786 after retry RPC after error@1324961912) -> (continuation of Deferred@571998136 after retry RPC after error@537190449) -> (continuation of Deferred@743024320 after retry RPC after error@683100577) -> (continuation of Deferred@893421872 after retry RPC after error@843248321), errback=(continuation of Deferred@795245636 after retry RPC after error@1904707019) -> (continuation of Deferred@1750162894 after retry RPC after error@1275047388) -> (continuation of Deferred@1474788539 after retry RPC after error@436408913) -> (continuation of Deferred@2027852786 after retry RPC after error@1324961912) -> (continuation of Deferred@571998136 after retry RPC after error@537190449) -> (continuation of Deferred@743024320 after retry RPC after error@683100577) -> (continuation of Deferred@893421872 after retry RPC after error@843248321)))
06:34:10.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.429 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.490 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1472236330(state=PENDING, result=null, callback=(continuation of Deferred@523902568 after retry RPC after error@1900933460) -> (continuation of Deferred@1756441616 after retry RPC after error@1006334967) -> (continuation of Deferred@1954407358 after retry RPC after error@1780775732) -> (continuation of Deferred@194255820 after retry RPC after error@1796253058) -> (continuation of Deferred@1894505293 after retry RPC after error@970974143) -> (continuation of Deferred@314496307 after retry RPC after error@2087476819) -> (continuation of Deferred@1391706912 after retry RPC after error@351736026), errback=(continuation of Deferred@523902568 after retry RPC after error@1900933460) -> (continuation of Deferred@1756441616 after retry RPC after error@1006334967) -> (continuation of Deferred@1954407358 after retry RPC after error@1780775732) -> (continuation of Deferred@194255820 after retry RPC after error@1796253058) -> (continuation of Deferred@1894505293 after retry RPC after error@970974143) -> (continuation of Deferred@314496307 after retry RPC after error@2087476819) -> (continuation of Deferred@1391706912 after retry RPC after error@351736026)))
06:34:10.490 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1472236330(state=PENDING, result=null, callback=(continuation of Deferred@523902568 after retry RPC after error@1900933460) -> (continuation of Deferred@1756441616 after retry RPC after error@1006334967) -> (continuation of Deferred@1954407358 after retry RPC after error@1780775732) -> (continuation of Deferred@194255820 after retry RPC after error@1796253058) -> (continuation of Deferred@1894505293 after retry RPC after error@970974143) -> (continuation of Deferred@314496307 after retry RPC after error@2087476819) -> (continuation of Deferred@1391706912 after retry RPC after error@351736026), errback=(continuation of Deferred@523902568 after retry RPC after error@1900933460) -> (continuation of Deferred@1756441616 after retry RPC after error@1006334967) -> (continuation of Deferred@1954407358 after retry RPC after error@1780775732) -> (continuation of Deferred@194255820 after retry RPC after error@1796253058) -> (continuation of Deferred@1894505293 after retry RPC after error@970974143) -> (continuation of Deferred@314496307 after retry RPC after error@2087476819) -> (continuation of Deferred@1391706912 after retry RPC after error@351736026)))
06:34:10.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.590 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.630 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.752 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@770226595(state=PENDING, result=null, callback=(continuation of Deferred@1987787455 after retry RPC after error@2116499440) -> (continuation of Deferred@1214803772 after retry RPC after error@539357747) -> (continuation of Deferred@262441546 after retry RPC after error@1224455417) -> (continuation of Deferred@1130439509 after retry RPC after error@1281025376) -> (continuation of Deferred@1728561275 after retry RPC after error@1926167582) -> (continuation of Deferred@947847833 after retry RPC after error@597000467) -> (continuation of Deferred@2051578114 after retry RPC after error@2137236569), errback=(continuation of Deferred@1987787455 after retry RPC after error@2116499440) -> (continuation of Deferred@1214803772 after retry RPC after error@539357747) -> (continuation of Deferred@262441546 after retry RPC after error@1224455417) -> (continuation of Deferred@1130439509 after retry RPC after error@1281025376) -> (continuation of Deferred@1728561275 after retry RPC after error@1926167582) -> (continuation of Deferred@947847833 after retry RPC after error@597000467) -> (continuation of Deferred@2051578114 after retry RPC after error@2137236569)))
06:34:10.753 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=243), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@770226595(state=PENDING, result=null, callback=(continuation of Deferred@1987787455 after retry RPC after error@2116499440) -> (continuation of Deferred@1214803772 after retry RPC after error@539357747) -> (continuation of Deferred@262441546 after retry RPC after error@1224455417) -> (continuation of Deferred@1130439509 after retry RPC after error@1281025376) -> (continuation of Deferred@1728561275 after retry RPC after error@1926167582) -> (continuation of Deferred@947847833 after retry RPC after error@597000467) -> (continuation of Deferred@2051578114 after retry RPC after error@2137236569), errback=(continuation of Deferred@1987787455 after retry RPC after error@2116499440) -> (continuation of Deferred@1214803772 after retry RPC after error@539357747) -> (continuation of Deferred@262441546 after retry RPC after error@1224455417) -> (continuation of Deferred@1130439509 after retry RPC after error@1281025376) -> (continuation of Deferred@1728561275 after retry RPC after error@1926167582) -> (continuation of Deferred@947847833 after retry RPC after error@597000467) -> (continuation of Deferred@2051578114 after retry RPC after error@2137236569)))
06:34:10.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.808 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.808 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.809 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1034128117(state=PENDING, result=null, callback=(continuation of Deferred@56205749 after retry RPC after error@1994557639) -> (continuation of Deferred@480186898 after retry RPC after error@1028310870) -> (continuation of Deferred@1943051587 after retry RPC after error@286693198) -> (continuation of Deferred@1979517180 after retry RPC after error@518390114) -> (continuation of Deferred@1772862840 after retry RPC after error@1277361124) -> (continuation of Deferred@1359323252 after retry RPC after error@2143700519), errback=(continuation of Deferred@56205749 after retry RPC after error@1994557639) -> (continuation of Deferred@480186898 after retry RPC after error@1028310870) -> (continuation of Deferred@1943051587 after retry RPC after error@286693198) -> (continuation of Deferred@1979517180 after retry RPC after error@518390114) -> (continuation of Deferred@1772862840 after retry RPC after error@1277361124) -> (continuation of Deferred@1359323252 after retry RPC after error@2143700519)))
06:34:10.951 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1034128117(state=PENDING, result=null, callback=(continuation of Deferred@56205749 after retry RPC after error@1994557639) -> (continuation of Deferred@480186898 after retry RPC after error@1028310870) -> (continuation of Deferred@1943051587 after retry RPC after error@286693198) -> (continuation of Deferred@1979517180 after retry RPC after error@518390114) -> (continuation of Deferred@1772862840 after retry RPC after error@1277361124) -> (continuation of Deferred@1359323252 after retry RPC after error@2143700519), errback=(continuation of Deferred@56205749 after retry RPC after error@1994557639) -> (continuation of Deferred@480186898 after retry RPC after error@1028310870) -> (continuation of Deferred@1943051587 after retry RPC after error@286693198) -> (continuation of Deferred@1979517180 after retry RPC after error@518390114) -> (continuation of Deferred@1772862840 after retry RPC after error@1277361124) -> (continuation of Deferred@1359323252 after retry RPC after error@2143700519)))
06:34:10.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:10.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:10.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:10.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:10.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:10.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1600224001(state=PENDING, result=null, callback=(continuation of Deferred@916666946 after retry RPC after error@1604360524) -> (continuation of Deferred@12824816 after retry RPC after error@1412332186) -> (continuation of Deferred@2070948727 after retry RPC after error@549566380) -> (continuation of Deferred@1033426804 after retry RPC after error@440766711) -> (continuation of Deferred@1620327259 after retry RPC after error@1296407602) -> (continuation of Deferred@907923865 after retry RPC after error@6892933), errback=(continuation of Deferred@916666946 after retry RPC after error@1604360524) -> (continuation of Deferred@12824816 after retry RPC after error@1412332186) -> (continuation of Deferred@2070948727 after retry RPC after error@549566380) -> (continuation of Deferred@1033426804 after retry RPC after error@440766711) -> (continuation of Deferred@1620327259 after retry RPC after error@1296407602) -> (continuation of Deferred@907923865 after retry RPC after error@6892933)))
06:34:11.150 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1600224001(state=PENDING, result=null, callback=(continuation of Deferred@916666946 after retry RPC after error@1604360524) -> (continuation of Deferred@12824816 after retry RPC after error@1412332186) -> (continuation of Deferred@2070948727 after retry RPC after error@549566380) -> (continuation of Deferred@1033426804 after retry RPC after error@440766711) -> (continuation of Deferred@1620327259 after retry RPC after error@1296407602) -> (continuation of Deferred@907923865 after retry RPC after error@6892933), errback=(continuation of Deferred@916666946 after retry RPC after error@1604360524) -> (continuation of Deferred@12824816 after retry RPC after error@1412332186) -> (continuation of Deferred@2070948727 after retry RPC after error@549566380) -> (continuation of Deferred@1033426804 after retry RPC after error@440766711) -> (continuation of Deferred@1620327259 after retry RPC after error@1296407602) -> (continuation of Deferred@907923865 after retry RPC after error@6892933)))
06:34:11.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.330 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.330 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.391 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1723352397(state=PENDING, result=null, callback=(continuation of Deferred@218827697 after retry RPC after error@1189497571) -> (continuation of Deferred@366965702 after retry RPC after error@837410333) -> (continuation of Deferred@1693060370 after retry RPC after error@271573566) -> (continuation of Deferred@2113987262 after retry RPC after error@23511637) -> (continuation of Deferred@902880758 after retry RPC after error@1518466912) -> (continuation of Deferred@602799004 after retry RPC after error@347005406) -> (continuation of Deferred@656585168 after retry RPC after error@1917675683), errback=(continuation of Deferred@218827697 after retry RPC after error@1189497571) -> (continuation of Deferred@366965702 after retry RPC after error@837410333) -> (continuation of Deferred@1693060370 after retry RPC after error@271573566) -> (continuation of Deferred@2113987262 after retry RPC after error@23511637) -> (continuation of Deferred@902880758 after retry RPC after error@1518466912) -> (continuation of Deferred@602799004 after retry RPC after error@347005406) -> (continuation of Deferred@656585168 after retry RPC after error@1917675683)))
06:34:11.392 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1723352397(state=PENDING, result=null, callback=(continuation of Deferred@218827697 after retry RPC after error@1189497571) -> (continuation of Deferred@366965702 after retry RPC after error@837410333) -> (continuation of Deferred@1693060370 after retry RPC after error@271573566) -> (continuation of Deferred@2113987262 after retry RPC after error@23511637) -> (continuation of Deferred@902880758 after retry RPC after error@1518466912) -> (continuation of Deferred@602799004 after retry RPC after error@347005406) -> (continuation of Deferred@656585168 after retry RPC after error@1917675683), errback=(continuation of Deferred@218827697 after retry RPC after error@1189497571) -> (continuation of Deferred@366965702 after retry RPC after error@837410333) -> (continuation of Deferred@1693060370 after retry RPC after error@271573566) -> (continuation of Deferred@2113987262 after retry RPC after error@23511637) -> (continuation of Deferred@902880758 after retry RPC after error@1518466912) -> (continuation of Deferred@602799004 after retry RPC after error@347005406) -> (continuation of Deferred@656585168 after retry RPC after error@1917675683)))
06:34:11.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.570 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1992705584(state=PENDING, result=null, callback=(continuation of Deferred@721914221 after retry RPC after error@148823644) -> (continuation of Deferred@1831974297 after retry RPC after error@688032201) -> (continuation of Deferred@1054421349 after retry RPC after error@1082860764) -> (continuation of Deferred@553045033 after retry RPC after error@487019839) -> (continuation of Deferred@1789009379 after retry RPC after error@1930848687) -> (continuation of Deferred@1704420448 after retry RPC after error@1109293888), errback=(continuation of Deferred@721914221 after retry RPC after error@148823644) -> (continuation of Deferred@1831974297 after retry RPC after error@688032201) -> (continuation of Deferred@1054421349 after retry RPC after error@1082860764) -> (continuation of Deferred@553045033 after retry RPC after error@487019839) -> (continuation of Deferred@1789009379 after retry RPC after error@1930848687) -> (continuation of Deferred@1704420448 after retry RPC after error@1109293888)))
06:34:11.572 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1992705584(state=PENDING, result=null, callback=(continuation of Deferred@721914221 after retry RPC after error@148823644) -> (continuation of Deferred@1831974297 after retry RPC after error@688032201) -> (continuation of Deferred@1054421349 after retry RPC after error@1082860764) -> (continuation of Deferred@553045033 after retry RPC after error@487019839) -> (continuation of Deferred@1789009379 after retry RPC after error@1930848687) -> (continuation of Deferred@1704420448 after retry RPC after error@1109293888), errback=(continuation of Deferred@721914221 after retry RPC after error@148823644) -> (continuation of Deferred@1831974297 after retry RPC after error@688032201) -> (continuation of Deferred@1054421349 after retry RPC after error@1082860764) -> (continuation of Deferred@553045033 after retry RPC after error@487019839) -> (continuation of Deferred@1789009379 after retry RPC after error@1930848687) -> (continuation of Deferred@1704420448 after retry RPC after error@1109293888)))
06:34:11.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.590 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.633 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.633 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.633 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.633 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.633 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.633 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.633 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.771 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1139503340(state=PENDING, result=null, callback=(continuation of Deferred@1482062017 after retry RPC after error@1768387950) -> (continuation of Deferred@2108133865 after retry RPC after error@1708389025) -> (continuation of Deferred@245894826 after retry RPC after error@1547093524) -> (continuation of Deferred@501169314 after retry RPC after error@1742967618) -> (continuation of Deferred@695209034 after retry RPC after error@1134328086) -> (continuation of Deferred@667052011 after retry RPC after error@1547737554), errback=(continuation of Deferred@1482062017 after retry RPC after error@1768387950) -> (continuation of Deferred@2108133865 after retry RPC after error@1708389025) -> (continuation of Deferred@245894826 after retry RPC after error@1547093524) -> (continuation of Deferred@501169314 after retry RPC after error@1742967618) -> (continuation of Deferred@695209034 after retry RPC after error@1134328086) -> (continuation of Deferred@667052011 after retry RPC after error@1547737554)))
06:34:11.771 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [45ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1139503340(state=PENDING, result=null, callback=(continuation of Deferred@1482062017 after retry RPC after error@1768387950) -> (continuation of Deferred@2108133865 after retry RPC after error@1708389025) -> (continuation of Deferred@245894826 after retry RPC after error@1547093524) -> (continuation of Deferred@501169314 after retry RPC after error@1742967618) -> (continuation of Deferred@695209034 after retry RPC after error@1134328086) -> (continuation of Deferred@667052011 after retry RPC after error@1547737554), errback=(continuation of Deferred@1482062017 after retry RPC after error@1768387950) -> (continuation of Deferred@2108133865 after retry RPC after error@1708389025) -> (continuation of Deferred@245894826 after retry RPC after error@1547093524) -> (continuation of Deferred@501169314 after retry RPC after error@1742967618) -> (continuation of Deferred@695209034 after retry RPC after error@1134328086) -> (continuation of Deferred@667052011 after retry RPC after error@1547737554)))
06:34:11.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.848 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.849 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.951 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@587297319(state=PENDING, result=null, callback=(continuation of Deferred@911299182 after retry RPC after error@1078152956) -> (continuation of Deferred@1213613784 after retry RPC after error@1626244543) -> (continuation of Deferred@1552972270 after retry RPC after error@108980270) -> (continuation of Deferred@2086936255 after retry RPC after error@1748221937) -> (continuation of Deferred@934536208 after retry RPC after error@1191007568) -> (continuation of Deferred@447757014 after retry RPC after error@1871299142), errback=(continuation of Deferred@911299182 after retry RPC after error@1078152956) -> (continuation of Deferred@1213613784 after retry RPC after error@1626244543) -> (continuation of Deferred@1552972270 after retry RPC after error@108980270) -> (continuation of Deferred@2086936255 after retry RPC after error@1748221937) -> (continuation of Deferred@934536208 after retry RPC after error@1191007568) -> (continuation of Deferred@447757014 after retry RPC after error@1871299142)))
06:34:11.951 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@587297319(state=PENDING, result=null, callback=(continuation of Deferred@911299182 after retry RPC after error@1078152956) -> (continuation of Deferred@1213613784 after retry RPC after error@1626244543) -> (continuation of Deferred@1552972270 after retry RPC after error@108980270) -> (continuation of Deferred@2086936255 after retry RPC after error@1748221937) -> (continuation of Deferred@934536208 after retry RPC after error@1191007568) -> (continuation of Deferred@447757014 after retry RPC after error@1871299142), errback=(continuation of Deferred@911299182 after retry RPC after error@1078152956) -> (continuation of Deferred@1213613784 after retry RPC after error@1626244543) -> (continuation of Deferred@1552972270 after retry RPC after error@108980270) -> (continuation of Deferred@2086936255 after retry RPC after error@1748221937) -> (continuation of Deferred@934536208 after retry RPC after error@1191007568) -> (continuation of Deferred@447757014 after retry RPC after error@1871299142)))
06:34:11.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.968 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.968 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:11.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:11.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:11.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:11.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:11.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.168 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.170 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1324243856(state=PENDING, result=null, callback=(continuation of Deferred@1987467080 after retry RPC after error@1229125566) -> (continuation of Deferred@1424637616 after retry RPC after error@1759829350) -> (continuation of Deferred@753666853 after retry RPC after error@1917707777) -> (continuation of Deferred@2002213190 after retry RPC after error@957785383) -> (continuation of Deferred@1970217987 after retry RPC after error@1754249537) -> (continuation of Deferred@627003519 after retry RPC after error@695002734) -> (continuation of Deferred@623870076 after retry RPC after error@822042665), errback=(continuation of Deferred@1987467080 after retry RPC after error@1229125566) -> (continuation of Deferred@1424637616 after retry RPC after error@1759829350) -> (continuation of Deferred@753666853 after retry RPC after error@1917707777) -> (continuation of Deferred@2002213190 after retry RPC after error@957785383) -> (continuation of Deferred@1970217987 after retry RPC after error@1754249537) -> (continuation of Deferred@627003519 after retry RPC after error@695002734) -> (continuation of Deferred@623870076 after retry RPC after error@822042665)))
06:34:12.170 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1324243856(state=PENDING, result=null, callback=(continuation of Deferred@1987467080 after retry RPC after error@1229125566) -> (continuation of Deferred@1424637616 after retry RPC after error@1759829350) -> (continuation of Deferred@753666853 after retry RPC after error@1917707777) -> (continuation of Deferred@2002213190 after retry RPC after error@957785383) -> (continuation of Deferred@1970217987 after retry RPC after error@1754249537) -> (continuation of Deferred@627003519 after retry RPC after error@695002734) -> (continuation of Deferred@623870076 after retry RPC after error@822042665), errback=(continuation of Deferred@1987467080 after retry RPC after error@1229125566) -> (continuation of Deferred@1424637616 after retry RPC after error@1759829350) -> (continuation of Deferred@753666853 after retry RPC after error@1917707777) -> (continuation of Deferred@2002213190 after retry RPC after error@957785383) -> (continuation of Deferred@1970217987 after retry RPC after error@1754249537) -> (continuation of Deferred@627003519 after retry RPC after error@695002734) -> (continuation of Deferred@623870076 after retry RPC after error@822042665)))
06:34:12.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.291 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:12.323958 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:12.324 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:12.324232 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:12.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@529712157(state=PENDING, result=null, callback=(continuation of Deferred@1867724978 after retry RPC after error@999233551) -> (continuation of Deferred@1838165879 after retry RPC after error@1680588020) -> (continuation of Deferred@315330155 after retry RPC after error@1363649373) -> (continuation of Deferred@1027655061 after retry RPC after error@78975701) -> (continuation of Deferred@1245936894 after retry RPC after error@653419424) -> (continuation of Deferred@497383046 after retry RPC after error@824900701), errback=(continuation of Deferred@1867724978 after retry RPC after error@999233551) -> (continuation of Deferred@1838165879 after retry RPC after error@1680588020) -> (continuation of Deferred@315330155 after retry RPC after error@1363649373) -> (continuation of Deferred@1027655061 after retry RPC after error@78975701) -> (continuation of Deferred@1245936894 after retry RPC after error@653419424) -> (continuation of Deferred@497383046 after retry RPC after error@824900701)))
06:34:12.371 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [103ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@529712157(state=PENDING, result=null, callback=(continuation of Deferred@1867724978 after retry RPC after error@999233551) -> (continuation of Deferred@1838165879 after retry RPC after error@1680588020) -> (continuation of Deferred@315330155 after retry RPC after error@1363649373) -> (continuation of Deferred@1027655061 after retry RPC after error@78975701) -> (continuation of Deferred@1245936894 after retry RPC after error@653419424) -> (continuation of Deferred@497383046 after retry RPC after error@824900701), errback=(continuation of Deferred@1867724978 after retry RPC after error@999233551) -> (continuation of Deferred@1838165879 after retry RPC after error@1680588020) -> (continuation of Deferred@315330155 after retry RPC after error@1363649373) -> (continuation of Deferred@1027655061 after retry RPC after error@78975701) -> (continuation of Deferred@1245936894 after retry RPC after error@653419424) -> (continuation of Deferred@497383046 after retry RPC after error@824900701)))
06:34:12.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1881555762(state=PENDING, result=null, callback=(continuation of Deferred@462423655 after retry RPC after error@1563544680) -> (continuation of Deferred@1857156637 after retry RPC after error@743184170) -> (continuation of Deferred@938778618 after retry RPC after error@1685411808) -> (continuation of Deferred@93668461 after retry RPC after error@1236068018) -> (continuation of Deferred@1521789746 after retry RPC after error@347094483) -> (continuation of Deferred@992589060 after retry RPC after error@362696101) -> (continuation of Deferred@1787595039 after retry RPC after error@2143780664), errback=(continuation of Deferred@462423655 after retry RPC after error@1563544680) -> (continuation of Deferred@1857156637 after retry RPC after error@743184170) -> (continuation of Deferred@938778618 after retry RPC after error@1685411808) -> (continuation of Deferred@93668461 after retry RPC after error@1236068018) -> (continuation of Deferred@1521789746 after retry RPC after error@347094483) -> (continuation of Deferred@992589060 after retry RPC after error@362696101) -> (continuation of Deferred@1787595039 after retry RPC after error@2143780664)))
06:34:12.611 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1881555762(state=PENDING, result=null, callback=(continuation of Deferred@462423655 after retry RPC after error@1563544680) -> (continuation of Deferred@1857156637 after retry RPC after error@743184170) -> (continuation of Deferred@938778618 after retry RPC after error@1685411808) -> (continuation of Deferred@93668461 after retry RPC after error@1236068018) -> (continuation of Deferred@1521789746 after retry RPC after error@347094483) -> (continuation of Deferred@992589060 after retry RPC after error@362696101) -> (continuation of Deferred@1787595039 after retry RPC after error@2143780664), errback=(continuation of Deferred@462423655 after retry RPC after error@1563544680) -> (continuation of Deferred@1857156637 after retry RPC after error@743184170) -> (continuation of Deferred@938778618 after retry RPC after error@1685411808) -> (continuation of Deferred@93668461 after retry RPC after error@1236068018) -> (continuation of Deferred@1521789746 after retry RPC after error@347094483) -> (continuation of Deferred@992589060 after retry RPC after error@362696101) -> (continuation of Deferred@1787595039 after retry RPC after error@2143780664)))
06:34:12.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.648 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.648 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.670 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.710 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.811 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@266011226(state=PENDING, result=null, callback=(continuation of Deferred@1863050331 after retry RPC after error@1007595884) -> (continuation of Deferred@1058715550 after retry RPC after error@1753525237) -> (continuation of Deferred@1486730114 after retry RPC after error@290581548) -> (continuation of Deferred@2065502175 after retry RPC after error@915130442) -> (continuation of Deferred@54492346 after retry RPC after error@496091261) -> (continuation of Deferred@249645031 after retry RPC after error@1723860969), errback=(continuation of Deferred@1863050331 after retry RPC after error@1007595884) -> (continuation of Deferred@1058715550 after retry RPC after error@1753525237) -> (continuation of Deferred@1486730114 after retry RPC after error@290581548) -> (continuation of Deferred@2065502175 after retry RPC after error@915130442) -> (continuation of Deferred@54492346 after retry RPC after error@496091261) -> (continuation of Deferred@249645031 after retry RPC after error@1723860969)))
06:34:12.811 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@266011226(state=PENDING, result=null, callback=(continuation of Deferred@1863050331 after retry RPC after error@1007595884) -> (continuation of Deferred@1058715550 after retry RPC after error@1753525237) -> (continuation of Deferred@1486730114 after retry RPC after error@290581548) -> (continuation of Deferred@2065502175 after retry RPC after error@915130442) -> (continuation of Deferred@54492346 after retry RPC after error@496091261) -> (continuation of Deferred@249645031 after retry RPC after error@1723860969), errback=(continuation of Deferred@1863050331 after retry RPC after error@1007595884) -> (continuation of Deferred@1058715550 after retry RPC after error@1753525237) -> (continuation of Deferred@1486730114 after retry RPC after error@290581548) -> (continuation of Deferred@2065502175 after retry RPC after error@915130442) -> (continuation of Deferred@54492346 after retry RPC after error@496091261) -> (continuation of Deferred@249645031 after retry RPC after error@1723860969)))
06:34:12.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.830 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.929 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:12.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:12.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:12.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:12.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:12.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.050 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.051 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.051 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [100ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1274909670(state=PENDING, result=null, callback=(continuation of Deferred@1095089723 after retry RPC after error@39376983) -> (continuation of Deferred@98068672 after retry RPC after error@580761918) -> (continuation of Deferred@1313177392 after retry RPC after error@177970383) -> (continuation of Deferred@1279490868 after retry RPC after error@56131531) -> (continuation of Deferred@1711352943 after retry RPC after error@808106372) -> (continuation of Deferred@1734868590 after retry RPC after error@662791022) -> (continuation of Deferred@1217151397 after retry RPC after error@755967913), errback=(continuation of Deferred@1095089723 after retry RPC after error@39376983) -> (continuation of Deferred@98068672 after retry RPC after error@580761918) -> (continuation of Deferred@1313177392 after retry RPC after error@177970383) -> (continuation of Deferred@1279490868 after retry RPC after error@56131531) -> (continuation of Deferred@1711352943 after retry RPC after error@808106372) -> (continuation of Deferred@1734868590 after retry RPC after error@662791022) -> (continuation of Deferred@1217151397 after retry RPC after error@755967913)))
06:34:13.052 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [100ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1274909670(state=PENDING, result=null, callback=(continuation of Deferred@1095089723 after retry RPC after error@39376983) -> (continuation of Deferred@98068672 after retry RPC after error@580761918) -> (continuation of Deferred@1313177392 after retry RPC after error@177970383) -> (continuation of Deferred@1279490868 after retry RPC after error@56131531) -> (continuation of Deferred@1711352943 after retry RPC after error@808106372) -> (continuation of Deferred@1734868590 after retry RPC after error@662791022) -> (continuation of Deferred@1217151397 after retry RPC after error@755967913), errback=(continuation of Deferred@1095089723 after retry RPC after error@39376983) -> (continuation of Deferred@98068672 after retry RPC after error@580761918) -> (continuation of Deferred@1313177392 after retry RPC after error@177970383) -> (continuation of Deferred@1279490868 after retry RPC after error@56131531) -> (continuation of Deferred@1711352943 after retry RPC after error@808106372) -> (continuation of Deferred@1734868590 after retry RPC after error@662791022) -> (continuation of Deferred@1217151397 after retry RPC after error@755967913)))
06:34:13.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.110 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.110 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.111 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.130 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.130 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.131 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.131 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.131 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.168 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.209 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.210 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.211 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.270 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.270 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.271 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@198686442(state=PENDING, result=null, callback=(continuation of Deferred@1574426788 after retry RPC after error@2086501297) -> (continuation of Deferred@1126649110 after retry RPC after error@1970389886) -> (continuation of Deferred@369108876 after retry RPC after error@18306574) -> (continuation of Deferred@2015491271 after retry RPC after error@61911694) -> (continuation of Deferred@345753907 after retry RPC after error@789006984) -> (continuation of Deferred@1484194915 after retry RPC after error@1591081249) -> (continuation of Deferred@349008281 after retry RPC after error@25901698), errback=(continuation of Deferred@1574426788 after retry RPC after error@2086501297) -> (continuation of Deferred@1126649110 after retry RPC after error@1970389886) -> (continuation of Deferred@369108876 after retry RPC after error@18306574) -> (continuation of Deferred@2015491271 after retry RPC after error@61911694) -> (continuation of Deferred@345753907 after retry RPC after error@789006984) -> (continuation of Deferred@1484194915 after retry RPC after error@1591081249) -> (continuation of Deferred@349008281 after retry RPC after error@25901698)))
06:34:13.272 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@198686442(state=PENDING, result=null, callback=(continuation of Deferred@1574426788 after retry RPC after error@2086501297) -> (continuation of Deferred@1126649110 after retry RPC after error@1970389886) -> (continuation of Deferred@369108876 after retry RPC after error@18306574) -> (continuation of Deferred@2015491271 after retry RPC after error@61911694) -> (continuation of Deferred@345753907 after retry RPC after error@789006984) -> (continuation of Deferred@1484194915 after retry RPC after error@1591081249) -> (continuation of Deferred@349008281 after retry RPC after error@25901698), errback=(continuation of Deferred@1574426788 after retry RPC after error@2086501297) -> (continuation of Deferred@1126649110 after retry RPC after error@1970389886) -> (continuation of Deferred@369108876 after retry RPC after error@18306574) -> (continuation of Deferred@2015491271 after retry RPC after error@61911694) -> (continuation of Deferred@345753907 after retry RPC after error@789006984) -> (continuation of Deferred@1484194915 after retry RPC after error@1591081249) -> (continuation of Deferred@349008281 after retry RPC after error@25901698)))
06:34:13.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.328 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.328 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.450 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.450 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.450 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@351211968(state=PENDING, result=null, callback=(continuation of Deferred@1416471245 after retry RPC after error@1702084829) -> (continuation of Deferred@2045754834 after retry RPC after error@358797886) -> (continuation of Deferred@748708245 after retry RPC after error@1462643188) -> (continuation of Deferred@1403719662 after retry RPC after error@1348191367) -> (continuation of Deferred@2136521680 after retry RPC after error@1495227462) -> (continuation of Deferred@1379990772 after retry RPC after error@1546741191), errback=(continuation of Deferred@1416471245 after retry RPC after error@1702084829) -> (continuation of Deferred@2045754834 after retry RPC after error@358797886) -> (continuation of Deferred@748708245 after retry RPC after error@1462643188) -> (continuation of Deferred@1403719662 after retry RPC after error@1348191367) -> (continuation of Deferred@2136521680 after retry RPC after error@1495227462) -> (continuation of Deferred@1379990772 after retry RPC after error@1546741191)))
06:34:13.451 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@351211968(state=PENDING, result=null, callback=(continuation of Deferred@1416471245 after retry RPC after error@1702084829) -> (continuation of Deferred@2045754834 after retry RPC after error@358797886) -> (continuation of Deferred@748708245 after retry RPC after error@1462643188) -> (continuation of Deferred@1403719662 after retry RPC after error@1348191367) -> (continuation of Deferred@2136521680 after retry RPC after error@1495227462) -> (continuation of Deferred@1379990772 after retry RPC after error@1546741191), errback=(continuation of Deferred@1416471245 after retry RPC after error@1702084829) -> (continuation of Deferred@2045754834 after retry RPC after error@358797886) -> (continuation of Deferred@748708245 after retry RPC after error@1462643188) -> (continuation of Deferred@1403719662 after retry RPC after error@1348191367) -> (continuation of Deferred@2136521680 after retry RPC after error@1495227462) -> (continuation of Deferred@1379990772 after retry RPC after error@1546741191)))
06:34:13.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.630 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.710 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.710 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.711 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@207556843(state=PENDING, result=null, callback=(continuation of Deferred@1689895177 after retry RPC after error@851852890) -> (continuation of Deferred@1695044725 after retry RPC after error@296015991) -> (continuation of Deferred@994046659 after retry RPC after error@948453136) -> (continuation of Deferred@528773000 after retry RPC after error@676625263) -> (continuation of Deferred@1824079180 after retry RPC after error@363921268) -> (continuation of Deferred@703867493 after retry RPC after error@586199863) -> (continuation of Deferred@117881208 after retry RPC after error@366984627), errback=(continuation of Deferred@1689895177 after retry RPC after error@851852890) -> (continuation of Deferred@1695044725 after retry RPC after error@296015991) -> (continuation of Deferred@994046659 after retry RPC after error@948453136) -> (continuation of Deferred@528773000 after retry RPC after error@676625263) -> (continuation of Deferred@1824079180 after retry RPC after error@363921268) -> (continuation of Deferred@703867493 after retry RPC after error@586199863) -> (continuation of Deferred@117881208 after retry RPC after error@366984627)))
06:34:13.712 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@207556843(state=PENDING, result=null, callback=(continuation of Deferred@1689895177 after retry RPC after error@851852890) -> (continuation of Deferred@1695044725 after retry RPC after error@296015991) -> (continuation of Deferred@994046659 after retry RPC after error@948453136) -> (continuation of Deferred@528773000 after retry RPC after error@676625263) -> (continuation of Deferred@1824079180 after retry RPC after error@363921268) -> (continuation of Deferred@703867493 after retry RPC after error@586199863) -> (continuation of Deferred@117881208 after retry RPC after error@366984627), errback=(continuation of Deferred@1689895177 after retry RPC after error@851852890) -> (continuation of Deferred@1695044725 after retry RPC after error@296015991) -> (continuation of Deferred@994046659 after retry RPC after error@948453136) -> (continuation of Deferred@528773000 after retry RPC after error@676625263) -> (continuation of Deferred@1824079180 after retry RPC after error@363921268) -> (continuation of Deferred@703867493 after retry RPC after error@586199863) -> (continuation of Deferred@117881208 after retry RPC after error@366984627)))
06:34:13.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.910 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.910 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.910 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.911 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.911 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@580264642(state=PENDING, result=null, callback=(continuation of Deferred@1509608404 after retry RPC after error@1287925878) -> (continuation of Deferred@1334579925 after retry RPC after error@1553971668) -> (continuation of Deferred@1395949048 after retry RPC after error@1042198632) -> (continuation of Deferred@296425412 after retry RPC after error@811289916) -> (continuation of Deferred@1081093490 after retry RPC after error@187062500) -> (continuation of Deferred@1123711859 after retry RPC after error@673381565) -> (continuation of Deferred@1769496438 after retry RPC after error@1352715205), errback=(continuation of Deferred@1509608404 after retry RPC after error@1287925878) -> (continuation of Deferred@1334579925 after retry RPC after error@1553971668) -> (continuation of Deferred@1395949048 after retry RPC after error@1042198632) -> (continuation of Deferred@296425412 after retry RPC after error@811289916) -> (continuation of Deferred@1081093490 after retry RPC after error@187062500) -> (continuation of Deferred@1123711859 after retry RPC after error@673381565) -> (continuation of Deferred@1769496438 after retry RPC after error@1352715205)))
06:34:13.912 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@580264642(state=PENDING, result=null, callback=(continuation of Deferred@1509608404 after retry RPC after error@1287925878) -> (continuation of Deferred@1334579925 after retry RPC after error@1553971668) -> (continuation of Deferred@1395949048 after retry RPC after error@1042198632) -> (continuation of Deferred@296425412 after retry RPC after error@811289916) -> (continuation of Deferred@1081093490 after retry RPC after error@187062500) -> (continuation of Deferred@1123711859 after retry RPC after error@673381565) -> (continuation of Deferred@1769496438 after retry RPC after error@1352715205), errback=(continuation of Deferred@1509608404 after retry RPC after error@1287925878) -> (continuation of Deferred@1334579925 after retry RPC after error@1553971668) -> (continuation of Deferred@1395949048 after retry RPC after error@1042198632) -> (continuation of Deferred@296425412 after retry RPC after error@811289916) -> (continuation of Deferred@1081093490 after retry RPC after error@187062500) -> (continuation of Deferred@1123711859 after retry RPC after error@673381565) -> (continuation of Deferred@1769496438 after retry RPC after error@1352715205)))
06:34:13.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:13.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:13.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:13.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:13.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:13.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.150 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@349293505(state=PENDING, result=null, callback=(continuation of Deferred@113880174 after retry RPC after error@858255058) -> (continuation of Deferred@127003785 after retry RPC after error@486842489) -> (continuation of Deferred@1457902955 after retry RPC after error@1513680923) -> (continuation of Deferred@1380301425 after retry RPC after error@1786437416) -> (continuation of Deferred@306975105 after retry RPC after error@266961915) -> (continuation of Deferred@1879057989 after retry RPC after error@1279210738) -> (continuation of Deferred@1049490669 after retry RPC after error@2100842140), errback=(continuation of Deferred@113880174 after retry RPC after error@858255058) -> (continuation of Deferred@127003785 after retry RPC after error@486842489) -> (continuation of Deferred@1457902955 after retry RPC after error@1513680923) -> (continuation of Deferred@1380301425 after retry RPC after error@1786437416) -> (continuation of Deferred@306975105 after retry RPC after error@266961915) -> (continuation of Deferred@1879057989 after retry RPC after error@1279210738) -> (continuation of Deferred@1049490669 after retry RPC after error@2100842140)))
06:34:14.150 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@349293505(state=PENDING, result=null, callback=(continuation of Deferred@113880174 after retry RPC after error@858255058) -> (continuation of Deferred@127003785 after retry RPC after error@486842489) -> (continuation of Deferred@1457902955 after retry RPC after error@1513680923) -> (continuation of Deferred@1380301425 after retry RPC after error@1786437416) -> (continuation of Deferred@306975105 after retry RPC after error@266961915) -> (continuation of Deferred@1879057989 after retry RPC after error@1279210738) -> (continuation of Deferred@1049490669 after retry RPC after error@2100842140), errback=(continuation of Deferred@113880174 after retry RPC after error@858255058) -> (continuation of Deferred@127003785 after retry RPC after error@486842489) -> (continuation of Deferred@1457902955 after retry RPC after error@1513680923) -> (continuation of Deferred@1380301425 after retry RPC after error@1786437416) -> (continuation of Deferred@306975105 after retry RPC after error@266961915) -> (continuation of Deferred@1879057989 after retry RPC after error@1279210738) -> (continuation of Deferred@1049490669 after retry RPC after error@2100842140)))
06:34:14.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.208 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1855560911(state=PENDING, result=null, callback=(continuation of Deferred@751040025 after retry RPC after error@1417603335) -> (continuation of Deferred@318463819 after retry RPC after error@816240191) -> (continuation of Deferred@57052428 after retry RPC after error@552823478) -> (continuation of Deferred@1757051108 after retry RPC after error@1848335586) -> (continuation of Deferred@1494397638 after retry RPC after error@1653589384) -> (continuation of Deferred@1506405477 after retry RPC after error@1831954418), errback=(continuation of Deferred@751040025 after retry RPC after error@1417603335) -> (continuation of Deferred@318463819 after retry RPC after error@816240191) -> (continuation of Deferred@57052428 after retry RPC after error@552823478) -> (continuation of Deferred@1757051108 after retry RPC after error@1848335586) -> (continuation of Deferred@1494397638 after retry RPC after error@1653589384) -> (continuation of Deferred@1506405477 after retry RPC after error@1831954418)))
06:34:14.331 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1855560911(state=PENDING, result=null, callback=(continuation of Deferred@751040025 after retry RPC after error@1417603335) -> (continuation of Deferred@318463819 after retry RPC after error@816240191) -> (continuation of Deferred@57052428 after retry RPC after error@552823478) -> (continuation of Deferred@1757051108 after retry RPC after error@1848335586) -> (continuation of Deferred@1494397638 after retry RPC after error@1653589384) -> (continuation of Deferred@1506405477 after retry RPC after error@1831954418), errback=(continuation of Deferred@751040025 after retry RPC after error@1417603335) -> (continuation of Deferred@318463819 after retry RPC after error@816240191) -> (continuation of Deferred@57052428 after retry RPC after error@552823478) -> (continuation of Deferred@1757051108 after retry RPC after error@1848335586) -> (continuation of Deferred@1494397638 after retry RPC after error@1653589384) -> (continuation of Deferred@1506405477 after retry RPC after error@1831954418)))
06:34:14.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.350 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.408 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.408 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.471 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.510 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.530 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [59ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [179ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1964190101(state=PENDING, result=null, callback=(continuation of Deferred@924796721 after retry RPC after error@968500706) -> (continuation of Deferred@1656715901 after retry RPC after error@244099134) -> (continuation of Deferred@1130840081 after retry RPC after error@723187882) -> (continuation of Deferred@854524710 after retry RPC after error@1497989149) -> (continuation of Deferred@550974701 after retry RPC after error@1559513077) -> (continuation of Deferred@382245246 after retry RPC after error@51690194) -> (continuation of Deferred@452822529 after retry RPC after error@1682481476), errback=(continuation of Deferred@924796721 after retry RPC after error@968500706) -> (continuation of Deferred@1656715901 after retry RPC after error@244099134) -> (continuation of Deferred@1130840081 after retry RPC after error@723187882) -> (continuation of Deferred@854524710 after retry RPC after error@1497989149) -> (continuation of Deferred@550974701 after retry RPC after error@1559513077) -> (continuation of Deferred@382245246 after retry RPC after error@51690194) -> (continuation of Deferred@452822529 after retry RPC after error@1682481476)))
06:34:14.530 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [59ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [179ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [181ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1964190101(state=PENDING, result=null, callback=(continuation of Deferred@924796721 after retry RPC after error@968500706) -> (continuation of Deferred@1656715901 after retry RPC after error@244099134) -> (continuation of Deferred@1130840081 after retry RPC after error@723187882) -> (continuation of Deferred@854524710 after retry RPC after error@1497989149) -> (continuation of Deferred@550974701 after retry RPC after error@1559513077) -> (continuation of Deferred@382245246 after retry RPC after error@51690194) -> (continuation of Deferred@452822529 after retry RPC after error@1682481476), errback=(continuation of Deferred@924796721 after retry RPC after error@968500706) -> (continuation of Deferred@1656715901 after retry RPC after error@244099134) -> (continuation of Deferred@1130840081 after retry RPC after error@723187882) -> (continuation of Deferred@854524710 after retry RPC after error@1497989149) -> (continuation of Deferred@550974701 after retry RPC after error@1559513077) -> (continuation of Deferred@382245246 after retry RPC after error@51690194) -> (continuation of Deferred@452822529 after retry RPC after error@1682481476)))
06:34:14.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.591 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.731 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@741742848(state=PENDING, result=null, callback=(continuation of Deferred@1595859803 after retry RPC after error@1179272665) -> (continuation of Deferred@1501165978 after retry RPC after error@1167787436) -> (continuation of Deferred@1832329855 after retry RPC after error@2043135629) -> (continuation of Deferred@1868842246 after retry RPC after error@1926204195) -> (continuation of Deferred@641690591 after retry RPC after error@1628123707) -> (continuation of Deferred@747247465 after retry RPC after error@579999560), errback=(continuation of Deferred@1595859803 after retry RPC after error@1179272665) -> (continuation of Deferred@1501165978 after retry RPC after error@1167787436) -> (continuation of Deferred@1832329855 after retry RPC after error@2043135629) -> (continuation of Deferred@1868842246 after retry RPC after error@1926204195) -> (continuation of Deferred@641690591 after retry RPC after error@1628123707) -> (continuation of Deferred@747247465 after retry RPC after error@579999560)))
06:34:14.731 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@741742848(state=PENDING, result=null, callback=(continuation of Deferred@1595859803 after retry RPC after error@1179272665) -> (continuation of Deferred@1501165978 after retry RPC after error@1167787436) -> (continuation of Deferred@1832329855 after retry RPC after error@2043135629) -> (continuation of Deferred@1868842246 after retry RPC after error@1926204195) -> (continuation of Deferred@641690591 after retry RPC after error@1628123707) -> (continuation of Deferred@747247465 after retry RPC after error@579999560), errback=(continuation of Deferred@1595859803 after retry RPC after error@1179272665) -> (continuation of Deferred@1501165978 after retry RPC after error@1167787436) -> (continuation of Deferred@1832329855 after retry RPC after error@2043135629) -> (continuation of Deferred@1868842246 after retry RPC after error@1926204195) -> (continuation of Deferred@641690591 after retry RPC after error@1628123707) -> (continuation of Deferred@747247465 after retry RPC after error@579999560)))
06:34:14.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.768 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.768 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.768 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.791 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.828 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:14.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:14.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:14.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:14.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:14.991 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1431751539(state=PENDING, result=null, callback=(continuation of Deferred@1597211407 after retry RPC after error@1693539031) -> (continuation of Deferred@1206144736 after retry RPC after error@613029948) -> (continuation of Deferred@1742916697 after retry RPC after error@162203024) -> (continuation of Deferred@1818089739 after retry RPC after error@409102148) -> (continuation of Deferred@1771298837 after retry RPC after error@944107314) -> (continuation of Deferred@978517168 after retry RPC after error@822211833) -> (continuation of Deferred@1197750660 after retry RPC after error@1347383845), errback=(continuation of Deferred@1597211407 after retry RPC after error@1693539031) -> (continuation of Deferred@1206144736 after retry RPC after error@613029948) -> (continuation of Deferred@1742916697 after retry RPC after error@162203024) -> (continuation of Deferred@1818089739 after retry RPC after error@409102148) -> (continuation of Deferred@1771298837 after retry RPC after error@944107314) -> (continuation of Deferred@978517168 after retry RPC after error@822211833) -> (continuation of Deferred@1197750660 after retry RPC after error@1347383845)))
06:34:14.992 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1431751539(state=PENDING, result=null, callback=(continuation of Deferred@1597211407 after retry RPC after error@1693539031) -> (continuation of Deferred@1206144736 after retry RPC after error@613029948) -> (continuation of Deferred@1742916697 after retry RPC after error@162203024) -> (continuation of Deferred@1818089739 after retry RPC after error@409102148) -> (continuation of Deferred@1771298837 after retry RPC after error@944107314) -> (continuation of Deferred@978517168 after retry RPC after error@822211833) -> (continuation of Deferred@1197750660 after retry RPC after error@1347383845), errback=(continuation of Deferred@1597211407 after retry RPC after error@1693539031) -> (continuation of Deferred@1206144736 after retry RPC after error@613029948) -> (continuation of Deferred@1742916697 after retry RPC after error@162203024) -> (continuation of Deferred@1818089739 after retry RPC after error@409102148) -> (continuation of Deferred@1771298837 after retry RPC after error@944107314) -> (continuation of Deferred@978517168 after retry RPC after error@822211833) -> (continuation of Deferred@1197750660 after retry RPC after error@1347383845)))
06:34:15.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.010 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.010 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.048 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.070 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.070 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.130 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.130 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.130 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.132 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.193 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:15.192896 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:15.197 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:15.197448 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:15.199 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:15.198951 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:15.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.210 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1364178820(state=PENDING, result=null, callback=(continuation of Deferred@1008927560 after retry RPC after error@1440482208) -> (continuation of Deferred@1092961658 after retry RPC after error@1836722509) -> (continuation of Deferred@1231507177 after retry RPC after error@2081658214) -> (continuation of Deferred@393609483 after retry RPC after error@729445097) -> (continuation of Deferred@1486446953 after retry RPC after error@952199097) -> (continuation of Deferred@521684979 after retry RPC after error@1728938947) -> (continuation of Deferred@1955345506 after retry RPC after error@1195600357), errback=(continuation of Deferred@1008927560 after retry RPC after error@1440482208) -> (continuation of Deferred@1092961658 after retry RPC after error@1836722509) -> (continuation of Deferred@1231507177 after retry RPC after error@2081658214) -> (continuation of Deferred@393609483 after retry RPC after error@729445097) -> (continuation of Deferred@1486446953 after retry RPC after error@952199097) -> (continuation of Deferred@521684979 after retry RPC after error@1728938947) -> (continuation of Deferred@1955345506 after retry RPC after error@1195600357)))
06:34:15.211 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [199ms] refreshing cache from master, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [199ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [201ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1364178820(state=PENDING, result=null, callback=(continuation of Deferred@1008927560 after retry RPC after error@1440482208) -> (continuation of Deferred@1092961658 after retry RPC after error@1836722509) -> (continuation of Deferred@1231507177 after retry RPC after error@2081658214) -> (continuation of Deferred@393609483 after retry RPC after error@729445097) -> (continuation of Deferred@1486446953 after retry RPC after error@952199097) -> (continuation of Deferred@521684979 after retry RPC after error@1728938947) -> (continuation of Deferred@1955345506 after retry RPC after error@1195600357), errback=(continuation of Deferred@1008927560 after retry RPC after error@1440482208) -> (continuation of Deferred@1092961658 after retry RPC after error@1836722509) -> (continuation of Deferred@1231507177 after retry RPC after error@2081658214) -> (continuation of Deferred@393609483 after retry RPC after error@729445097) -> (continuation of Deferred@1486446953 after retry RPC after error@952199097) -> (continuation of Deferred@521684979 after retry RPC after error@1728938947) -> (continuation of Deferred@1955345506 after retry RPC after error@1195600357)))
06:34:15.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.230 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.328 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.328 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.328 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.329 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.370 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.370 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.370 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.370 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.371 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@980799834(state=PENDING, result=null, callback=(continuation of Deferred@1134984075 after retry RPC after error@895647946) -> (continuation of Deferred@1371217858 after retry RPC after error@1162935214) -> (continuation of Deferred@903527084 after retry RPC after error@345966605) -> (continuation of Deferred@500383827 after retry RPC after error@1595688396) -> (continuation of Deferred@1072929625 after retry RPC after error@1405105098) -> (continuation of Deferred@1419128889 after retry RPC after error@1763261762), errback=(continuation of Deferred@1134984075 after retry RPC after error@895647946) -> (continuation of Deferred@1371217858 after retry RPC after error@1162935214) -> (continuation of Deferred@903527084 after retry RPC after error@345966605) -> (continuation of Deferred@500383827 after retry RPC after error@1595688396) -> (continuation of Deferred@1072929625 after retry RPC after error@1405105098) -> (continuation of Deferred@1419128889 after retry RPC after error@1763261762)))
06:34:15.372 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@980799834(state=PENDING, result=null, callback=(continuation of Deferred@1134984075 after retry RPC after error@895647946) -> (continuation of Deferred@1371217858 after retry RPC after error@1162935214) -> (continuation of Deferred@903527084 after retry RPC after error@345966605) -> (continuation of Deferred@500383827 after retry RPC after error@1595688396) -> (continuation of Deferred@1072929625 after retry RPC after error@1405105098) -> (continuation of Deferred@1419128889 after retry RPC after error@1763261762), errback=(continuation of Deferred@1134984075 after retry RPC after error@895647946) -> (continuation of Deferred@1371217858 after retry RPC after error@1162935214) -> (continuation of Deferred@903527084 after retry RPC after error@345966605) -> (continuation of Deferred@500383827 after retry RPC after error@1595688396) -> (continuation of Deferred@1072929625 after retry RPC after error@1405105098) -> (continuation of Deferred@1419128889 after retry RPC after error@1763261762)))
06:34:15.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.408 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.431 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.648 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@232519588(state=PENDING, result=null, callback=(continuation of Deferred@2066454263 after retry RPC after error@1867215074) -> (continuation of Deferred@286968966 after retry RPC after error@1135509462) -> (continuation of Deferred@1679152506 after retry RPC after error@1349346682) -> (continuation of Deferred@399830285 after retry RPC after error@657744110) -> (continuation of Deferred@1841356663 after retry RPC after error@1455235328) -> (continuation of Deferred@1714491999 after retry RPC after error@1652964711) -> (continuation of Deferred@165051687 after retry RPC after error@920451132) -> (continuation of Deferred@1485944658 after retry RPC after error@187679778), errback=(continuation of Deferred@2066454263 after retry RPC after error@1867215074) -> (continuation of Deferred@286968966 after retry RPC after error@1135509462) -> (continuation of Deferred@1679152506 after retry RPC after error@1349346682) -> (continuation of Deferred@399830285 after retry RPC after error@657744110) -> (continuation of Deferred@1841356663 after retry RPC after error@1455235328) -> (continuation of Deferred@1714491999 after retry RPC after error@1652964711) -> (continuation of Deferred@165051687 after retry RPC after error@920451132) -> (continuation of Deferred@1485944658 after retry RPC after error@187679778)))
06:34:15.649 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:15.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.670 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.789 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.790 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.790 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.850 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.851 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [39ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1324361878(state=PENDING, result=null, callback=(continuation of Deferred@312950845 after retry RPC after error@1441258899) -> (continuation of Deferred@794947327 after retry RPC after error@1612371741) -> (continuation of Deferred@1217713547 after retry RPC after error@594842191) -> (continuation of Deferred@63080136 after retry RPC after error@903224210) -> (continuation of Deferred@1526773806 after retry RPC after error@1558298603) -> (continuation of Deferred@1258456697 after retry RPC after error@1309478400), errback=(continuation of Deferred@312950845 after retry RPC after error@1441258899) -> (continuation of Deferred@794947327 after retry RPC after error@1612371741) -> (continuation of Deferred@1217713547 after retry RPC after error@594842191) -> (continuation of Deferred@63080136 after retry RPC after error@903224210) -> (continuation of Deferred@1526773806 after retry RPC after error@1558298603) -> (continuation of Deferred@1258456697 after retry RPC after error@1309478400)))
06:34:15.852 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [39ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [179ms] refreshing cache from master, [179ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [180ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1324361878(state=PENDING, result=null, callback=(continuation of Deferred@312950845 after retry RPC after error@1441258899) -> (continuation of Deferred@794947327 after retry RPC after error@1612371741) -> (continuation of Deferred@1217713547 after retry RPC after error@594842191) -> (continuation of Deferred@63080136 after retry RPC after error@903224210) -> (continuation of Deferred@1526773806 after retry RPC after error@1558298603) -> (continuation of Deferred@1258456697 after retry RPC after error@1309478400), errback=(continuation of Deferred@312950845 after retry RPC after error@1441258899) -> (continuation of Deferred@794947327 after retry RPC after error@1612371741) -> (continuation of Deferred@1217713547 after retry RPC after error@594842191) -> (continuation of Deferred@63080136 after retry RPC after error@903224210) -> (continuation of Deferred@1526773806 after retry RPC after error@1558298603) -> (continuation of Deferred@1258456697 after retry RPC after error@1309478400)))
06:34:15.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.929 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.968 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:15.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:15.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:15.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.990 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.990 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:15.990 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.990 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:15.991 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.128 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1791115460(state=PENDING, result=null, callback=(continuation of Deferred@157408859 after retry RPC after error@573968673) -> (continuation of Deferred@1646830832 after retry RPC after error@445910033) -> (continuation of Deferred@494967326 after retry RPC after error@263008150) -> (continuation of Deferred@679806391 after retry RPC after error@315322418) -> (continuation of Deferred@184703007 after retry RPC after error@1840584566) -> (continuation of Deferred@1830621130 after retry RPC after error@614377659) -> (continuation of Deferred@1027989143 after retry RPC after error@2022596846), errback=(continuation of Deferred@157408859 after retry RPC after error@573968673) -> (continuation of Deferred@1646830832 after retry RPC after error@445910033) -> (continuation of Deferred@494967326 after retry RPC after error@263008150) -> (continuation of Deferred@679806391 after retry RPC after error@315322418) -> (continuation of Deferred@184703007 after retry RPC after error@1840584566) -> (continuation of Deferred@1830621130 after retry RPC after error@614377659) -> (continuation of Deferred@1027989143 after retry RPC after error@2022596846)))
06:34:16.129 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:16.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.208 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.208 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1483948750(state=PENDING, result=null, callback=(continuation of Deferred@1818250267 after retry RPC after error@994383364) -> (continuation of Deferred@1580055280 after retry RPC after error@478865097) -> (continuation of Deferred@1927569879 after retry RPC after error@1627569337) -> (continuation of Deferred@1389060179 after retry RPC after error@431777901) -> (continuation of Deferred@1216829524 after retry RPC after error@1016386185) -> (continuation of Deferred@903144624 after retry RPC after error@1339100519) -> (continuation of Deferred@1030142257 after retry RPC after error@670839950), errback=(continuation of Deferred@1818250267 after retry RPC after error@994383364) -> (continuation of Deferred@1580055280 after retry RPC after error@478865097) -> (continuation of Deferred@1927569879 after retry RPC after error@1627569337) -> (continuation of Deferred@1389060179 after retry RPC after error@431777901) -> (continuation of Deferred@1216829524 after retry RPC after error@1016386185) -> (continuation of Deferred@903144624 after retry RPC after error@1339100519) -> (continuation of Deferred@1030142257 after retry RPC after error@670839950)))
06:34:16.331 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1483948750(state=PENDING, result=null, callback=(continuation of Deferred@1818250267 after retry RPC after error@994383364) -> (continuation of Deferred@1580055280 after retry RPC after error@478865097) -> (continuation of Deferred@1927569879 after retry RPC after error@1627569337) -> (continuation of Deferred@1389060179 after retry RPC after error@431777901) -> (continuation of Deferred@1216829524 after retry RPC after error@1016386185) -> (continuation of Deferred@903144624 after retry RPC after error@1339100519) -> (continuation of Deferred@1030142257 after retry RPC after error@670839950), errback=(continuation of Deferred@1818250267 after retry RPC after error@994383364) -> (continuation of Deferred@1580055280 after retry RPC after error@478865097) -> (continuation of Deferred@1927569879 after retry RPC after error@1627569337) -> (continuation of Deferred@1389060179 after retry RPC after error@431777901) -> (continuation of Deferred@1216829524 after retry RPC after error@1016386185) -> (continuation of Deferred@903144624 after retry RPC after error@1339100519) -> (continuation of Deferred@1030142257 after retry RPC after error@670839950)))
06:34:16.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.591 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1657702334(state=PENDING, result=null, callback=(continuation of Deferred@371238566 after retry RPC after error@1803137120) -> (continuation of Deferred@820823133 after retry RPC after error@638142089) -> (continuation of Deferred@898613624 after retry RPC after error@324246809) -> (continuation of Deferred@1170959618 after retry RPC after error@697976380) -> (continuation of Deferred@1658181238 after retry RPC after error@1835086944) -> (continuation of Deferred@1935468467 after retry RPC after error@143455298) -> (continuation of Deferred@1143399747 after retry RPC after error@545266579) -> (continuation of Deferred@888471887 after retry RPC after error@1530187127), errback=(continuation of Deferred@371238566 after retry RPC after error@1803137120) -> (continuation of Deferred@820823133 after retry RPC after error@638142089) -> (continuation of Deferred@898613624 after retry RPC after error@324246809) -> (continuation of Deferred@1170959618 after retry RPC after error@697976380) -> (continuation of Deferred@1658181238 after retry RPC after error@1835086944) -> (continuation of Deferred@1935468467 after retry RPC after error@143455298) -> (continuation of Deferred@1143399747 after retry RPC after error@545266579) -> (continuation of Deferred@888471887 after retry RPC after error@1530187127)))
06:34:16.591 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1657702334(state=PENDING, result=null, callback=(continuation of Deferred@371238566 after retry RPC after error@1803137120) -> (continuation of Deferred@820823133 after retry RPC after error@638142089) -> (continuation of Deferred@898613624 after retry RPC after error@324246809) -> (continuation of Deferred@1170959618 after retry RPC after error@697976380) -> (continuation of Deferred@1658181238 after retry RPC after error@1835086944) -> (continuation of Deferred@1935468467 after retry RPC after error@143455298) -> (continuation of Deferred@1143399747 after retry RPC after error@545266579) -> (continuation of Deferred@888471887 after retry RPC after error@1530187127), errback=(continuation of Deferred@371238566 after retry RPC after error@1803137120) -> (continuation of Deferred@820823133 after retry RPC after error@638142089) -> (continuation of Deferred@898613624 after retry RPC after error@324246809) -> (continuation of Deferred@1170959618 after retry RPC after error@697976380) -> (continuation of Deferred@1658181238 after retry RPC after error@1835086944) -> (continuation of Deferred@1935468467 after retry RPC after error@143455298) -> (continuation of Deferred@1143399747 after retry RPC after error@545266579) -> (continuation of Deferred@888471887 after retry RPC after error@1530187127)))
06:34:16.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.608 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@67352884(state=PENDING, result=null, callback=(continuation of Deferred@461082727 after retry RPC after error@2008758177) -> (continuation of Deferred@688468913 after retry RPC after error@811267517) -> (continuation of Deferred@2003149689 after retry RPC after error@1506640907) -> (continuation of Deferred@968466488 after retry RPC after error@1588157871) -> (continuation of Deferred@67978088 after retry RPC after error@836067902) -> (continuation of Deferred@1159310428 after retry RPC after error@1363793269) -> (continuation of Deferred@1068620343 after retry RPC after error@628067181), errback=(continuation of Deferred@461082727 after retry RPC after error@2008758177) -> (continuation of Deferred@688468913 after retry RPC after error@811267517) -> (continuation of Deferred@2003149689 after retry RPC after error@1506640907) -> (continuation of Deferred@968466488 after retry RPC after error@1588157871) -> (continuation of Deferred@67978088 after retry RPC after error@836067902) -> (continuation of Deferred@1159310428 after retry RPC after error@1363793269) -> (continuation of Deferred@1068620343 after retry RPC after error@628067181)))
06:34:16.831 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@67352884(state=PENDING, result=null, callback=(continuation of Deferred@461082727 after retry RPC after error@2008758177) -> (continuation of Deferred@688468913 after retry RPC after error@811267517) -> (continuation of Deferred@2003149689 after retry RPC after error@1506640907) -> (continuation of Deferred@968466488 after retry RPC after error@1588157871) -> (continuation of Deferred@67978088 after retry RPC after error@836067902) -> (continuation of Deferred@1159310428 after retry RPC after error@1363793269) -> (continuation of Deferred@1068620343 after retry RPC after error@628067181), errback=(continuation of Deferred@461082727 after retry RPC after error@2008758177) -> (continuation of Deferred@688468913 after retry RPC after error@811267517) -> (continuation of Deferred@2003149689 after retry RPC after error@1506640907) -> (continuation of Deferred@968466488 after retry RPC after error@1588157871) -> (continuation of Deferred@67978088 after retry RPC after error@836067902) -> (continuation of Deferred@1159310428 after retry RPC after error@1363793269) -> (continuation of Deferred@1068620343 after retry RPC after error@628067181)))
06:34:16.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.848 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.848 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.848 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.849 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.888 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.888 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:16.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:16.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:16.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:16.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:16.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.109 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@84397487(state=PENDING, result=null, callback=(continuation of Deferred@621796982 after retry RPC after error@1995695936) -> (continuation of Deferred@1456653308 after retry RPC after error@1828918844) -> (continuation of Deferred@908368260 after retry RPC after error@1594143207) -> (continuation of Deferred@520701144 after retry RPC after error@1783497715) -> (continuation of Deferred@2025082759 after retry RPC after error@46486271) -> (continuation of Deferred@1472199163 after retry RPC after error@1299462500) -> (continuation of Deferred@400990010 after retry RPC after error@1067218323), errback=(continuation of Deferred@621796982 after retry RPC after error@1995695936) -> (continuation of Deferred@1456653308 after retry RPC after error@1828918844) -> (continuation of Deferred@908368260 after retry RPC after error@1594143207) -> (continuation of Deferred@520701144 after retry RPC after error@1783497715) -> (continuation of Deferred@2025082759 after retry RPC after error@46486271) -> (continuation of Deferred@1472199163 after retry RPC after error@1299462500) -> (continuation of Deferred@400990010 after retry RPC after error@1067218323)))
06:34:17.109 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:17.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.131 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.208 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.290 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.325 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:17.325656 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:17.326 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:17.325914 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:17.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.331 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1289848129(state=PENDING, result=null, callback=(continuation of Deferred@1259111605 after retry RPC after error@2076081583) -> (continuation of Deferred@271935973 after retry RPC after error@1346256175) -> (continuation of Deferred@2142778033 after retry RPC after error@2003008492) -> (continuation of Deferred@1277316772 after retry RPC after error@1834276170) -> (continuation of Deferred@1397186539 after retry RPC after error@1066918516) -> (continuation of Deferred@1637796940 after retry RPC after error@707505072) -> (continuation of Deferred@1132069145 after retry RPC after error@1140772672), errback=(continuation of Deferred@1259111605 after retry RPC after error@2076081583) -> (continuation of Deferred@271935973 after retry RPC after error@1346256175) -> (continuation of Deferred@2142778033 after retry RPC after error@2003008492) -> (continuation of Deferred@1277316772 after retry RPC after error@1834276170) -> (continuation of Deferred@1397186539 after retry RPC after error@1066918516) -> (continuation of Deferred@1637796940 after retry RPC after error@707505072) -> (continuation of Deferred@1132069145 after retry RPC after error@1140772672)))
06:34:17.331 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1289848129(state=PENDING, result=null, callback=(continuation of Deferred@1259111605 after retry RPC after error@2076081583) -> (continuation of Deferred@271935973 after retry RPC after error@1346256175) -> (continuation of Deferred@2142778033 after retry RPC after error@2003008492) -> (continuation of Deferred@1277316772 after retry RPC after error@1834276170) -> (continuation of Deferred@1397186539 after retry RPC after error@1066918516) -> (continuation of Deferred@1637796940 after retry RPC after error@707505072) -> (continuation of Deferred@1132069145 after retry RPC after error@1140772672), errback=(continuation of Deferred@1259111605 after retry RPC after error@2076081583) -> (continuation of Deferred@271935973 after retry RPC after error@1346256175) -> (continuation of Deferred@2142778033 after retry RPC after error@2003008492) -> (continuation of Deferred@1277316772 after retry RPC after error@1834276170) -> (continuation of Deferred@1397186539 after retry RPC after error@1066918516) -> (continuation of Deferred@1637796940 after retry RPC after error@707505072) -> (continuation of Deferred@1132069145 after retry RPC after error@1140772672)))
06:34:17.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.410 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.410 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.411 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.448 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.609 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@427151064(state=PENDING, result=null, callback=(continuation of Deferred@1158297041 after retry RPC after error@1997723144) -> (continuation of Deferred@1380649150 after retry RPC after error@1711218465) -> (continuation of Deferred@1909394162 after retry RPC after error@277618773) -> (continuation of Deferred@436786178 after retry RPC after error@127129205) -> (continuation of Deferred@243455215 after retry RPC after error@2138948578) -> (continuation of Deferred@587837643 after retry RPC after error@984672929) -> (continuation of Deferred@571307188 after retry RPC after error@863945156), errback=(continuation of Deferred@1158297041 after retry RPC after error@1997723144) -> (continuation of Deferred@1380649150 after retry RPC after error@1711218465) -> (continuation of Deferred@1909394162 after retry RPC after error@277618773) -> (continuation of Deferred@436786178 after retry RPC after error@127129205) -> (continuation of Deferred@243455215 after retry RPC after error@2138948578) -> (continuation of Deferred@587837643 after retry RPC after error@984672929) -> (continuation of Deferred@571307188 after retry RPC after error@863945156)))
06:34:17.609 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:17.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.689 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.871 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@624127910(state=PENDING, result=null, callback=(continuation of Deferred@1284775603 after retry RPC after error@1885422358) -> (continuation of Deferred@1835009357 after retry RPC after error@1401952536) -> (continuation of Deferred@1067312223 after retry RPC after error@1168013641) -> (continuation of Deferred@683230938 after retry RPC after error@2075316748) -> (continuation of Deferred@727543353 after retry RPC after error@490675754) -> (continuation of Deferred@2144474305 after retry RPC after error@312318853) -> (continuation of Deferred@1391526729 after retry RPC after error@928684403), errback=(continuation of Deferred@1284775603 after retry RPC after error@1885422358) -> (continuation of Deferred@1835009357 after retry RPC after error@1401952536) -> (continuation of Deferred@1067312223 after retry RPC after error@1168013641) -> (continuation of Deferred@683230938 after retry RPC after error@2075316748) -> (continuation of Deferred@727543353 after retry RPC after error@490675754) -> (continuation of Deferred@2144474305 after retry RPC after error@312318853) -> (continuation of Deferred@1391526729 after retry RPC after error@928684403)))
06:34:17.871 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@624127910(state=PENDING, result=null, callback=(continuation of Deferred@1284775603 after retry RPC after error@1885422358) -> (continuation of Deferred@1835009357 after retry RPC after error@1401952536) -> (continuation of Deferred@1067312223 after retry RPC after error@1168013641) -> (continuation of Deferred@683230938 after retry RPC after error@2075316748) -> (continuation of Deferred@727543353 after retry RPC after error@490675754) -> (continuation of Deferred@2144474305 after retry RPC after error@312318853) -> (continuation of Deferred@1391526729 after retry RPC after error@928684403), errback=(continuation of Deferred@1284775603 after retry RPC after error@1885422358) -> (continuation of Deferred@1835009357 after retry RPC after error@1401952536) -> (continuation of Deferred@1067312223 after retry RPC after error@1168013641) -> (continuation of Deferred@683230938 after retry RPC after error@2075316748) -> (continuation of Deferred@727543353 after retry RPC after error@490675754) -> (continuation of Deferred@2144474305 after retry RPC after error@312318853) -> (continuation of Deferred@1391526729 after retry RPC after error@928684403)))
06:34:17.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.890 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.928 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.928 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:17.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:17.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:17.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:17.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:17.969 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.008 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.050 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.050 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.050 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.050 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.051 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.071 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.071 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2036886431(state=PENDING, result=null, callback=(continuation of Deferred@821966064 after retry RPC after error@930044860) -> (continuation of Deferred@547164435 after retry RPC after error@445886988) -> (continuation of Deferred@1527471130 after retry RPC after error@385109628) -> (continuation of Deferred@1579794488 after retry RPC after error@1504516643) -> (continuation of Deferred@1489462912 after retry RPC after error@1819558986) -> (continuation of Deferred@2085380604 after retry RPC after error@1300303956) -> (continuation of Deferred@772905986 after retry RPC after error@1619643294), errback=(continuation of Deferred@821966064 after retry RPC after error@930044860) -> (continuation of Deferred@547164435 after retry RPC after error@445886988) -> (continuation of Deferred@1527471130 after retry RPC after error@385109628) -> (continuation of Deferred@1579794488 after retry RPC after error@1504516643) -> (continuation of Deferred@1489462912 after retry RPC after error@1819558986) -> (continuation of Deferred@2085380604 after retry RPC after error@1300303956) -> (continuation of Deferred@772905986 after retry RPC after error@1619643294)))
06:34:18.072 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2036886431(state=PENDING, result=null, callback=(continuation of Deferred@821966064 after retry RPC after error@930044860) -> (continuation of Deferred@547164435 after retry RPC after error@445886988) -> (continuation of Deferred@1527471130 after retry RPC after error@385109628) -> (continuation of Deferred@1579794488 after retry RPC after error@1504516643) -> (continuation of Deferred@1489462912 after retry RPC after error@1819558986) -> (continuation of Deferred@2085380604 after retry RPC after error@1300303956) -> (continuation of Deferred@772905986 after retry RPC after error@1619643294), errback=(continuation of Deferred@821966064 after retry RPC after error@930044860) -> (continuation of Deferred@547164435 after retry RPC after error@445886988) -> (continuation of Deferred@1527471130 after retry RPC after error@385109628) -> (continuation of Deferred@1579794488 after retry RPC after error@1504516643) -> (continuation of Deferred@1489462912 after retry RPC after error@1819558986) -> (continuation of Deferred@2085380604 after retry RPC after error@1300303956) -> (continuation of Deferred@772905986 after retry RPC after error@1619643294)))
06:34:18.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.108 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.108 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.108 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.108 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.109 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.148 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.148 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.148 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.188 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.309 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.310 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.310 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.310 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.311 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.330 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@891893055(state=PENDING, result=null, callback=(continuation of Deferred@1036637363 after retry RPC after error@1236324079) -> (continuation of Deferred@1535151950 after retry RPC after error@670878156) -> (continuation of Deferred@1415839180 after retry RPC after error@460031513) -> (continuation of Deferred@1154388165 after retry RPC after error@87921465) -> (continuation of Deferred@1325110223 after retry RPC after error@157785927) -> (continuation of Deferred@966749718 after retry RPC after error@1347585607) -> (continuation of Deferred@1109725801 after retry RPC after error@2005741773) -> (continuation of Deferred@891297780 after retry RPC after error@1896142761), errback=(continuation of Deferred@1036637363 after retry RPC after error@1236324079) -> (continuation of Deferred@1535151950 after retry RPC after error@670878156) -> (continuation of Deferred@1415839180 after retry RPC after error@460031513) -> (continuation of Deferred@1154388165 after retry RPC after error@87921465) -> (continuation of Deferred@1325110223 after retry RPC after error@157785927) -> (continuation of Deferred@966749718 after retry RPC after error@1347585607) -> (continuation of Deferred@1109725801 after retry RPC after error@2005741773) -> (continuation of Deferred@891297780 after retry RPC after error@1896142761)))
06:34:18.331 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [223ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@891893055(state=PENDING, result=null, callback=(continuation of Deferred@1036637363 after retry RPC after error@1236324079) -> (continuation of Deferred@1535151950 after retry RPC after error@670878156) -> (continuation of Deferred@1415839180 after retry RPC after error@460031513) -> (continuation of Deferred@1154388165 after retry RPC after error@87921465) -> (continuation of Deferred@1325110223 after retry RPC after error@157785927) -> (continuation of Deferred@966749718 after retry RPC after error@1347585607) -> (continuation of Deferred@1109725801 after retry RPC after error@2005741773) -> (continuation of Deferred@891297780 after retry RPC after error@1896142761), errback=(continuation of Deferred@1036637363 after retry RPC after error@1236324079) -> (continuation of Deferred@1535151950 after retry RPC after error@670878156) -> (continuation of Deferred@1415839180 after retry RPC after error@460031513) -> (continuation of Deferred@1154388165 after retry RPC after error@87921465) -> (continuation of Deferred@1325110223 after retry RPC after error@157785927) -> (continuation of Deferred@966749718 after retry RPC after error@1347585607) -> (continuation of Deferred@1109725801 after retry RPC after error@2005741773) -> (continuation of Deferred@891297780 after retry RPC after error@1896142761)))
06:34:18.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.348 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.450 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.512 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.551 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@219528769(state=PENDING, result=null, callback=(continuation of Deferred@1470038215 after retry RPC after error@1535563891) -> (continuation of Deferred@875169483 after retry RPC after error@342708665) -> (continuation of Deferred@1004266187 after retry RPC after error@1450438787) -> (continuation of Deferred@839629016 after retry RPC after error@1276495801) -> (continuation of Deferred@683452746 after retry RPC after error@1635951414) -> (continuation of Deferred@236984998 after retry RPC after error@1818901716) -> (continuation of Deferred@567361258 after retry RPC after error@1430172405), errback=(continuation of Deferred@1470038215 after retry RPC after error@1535563891) -> (continuation of Deferred@875169483 after retry RPC after error@342708665) -> (continuation of Deferred@1004266187 after retry RPC after error@1450438787) -> (continuation of Deferred@839629016 after retry RPC after error@1276495801) -> (continuation of Deferred@683452746 after retry RPC after error@1635951414) -> (continuation of Deferred@236984998 after retry RPC after error@1818901716) -> (continuation of Deferred@567361258 after retry RPC after error@1430172405)))
06:34:18.552 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=203), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [164ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [203ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@219528769(state=PENDING, result=null, callback=(continuation of Deferred@1470038215 after retry RPC after error@1535563891) -> (continuation of Deferred@875169483 after retry RPC after error@342708665) -> (continuation of Deferred@1004266187 after retry RPC after error@1450438787) -> (continuation of Deferred@839629016 after retry RPC after error@1276495801) -> (continuation of Deferred@683452746 after retry RPC after error@1635951414) -> (continuation of Deferred@236984998 after retry RPC after error@1818901716) -> (continuation of Deferred@567361258 after retry RPC after error@1430172405), errback=(continuation of Deferred@1470038215 after retry RPC after error@1535563891) -> (continuation of Deferred@875169483 after retry RPC after error@342708665) -> (continuation of Deferred@1004266187 after retry RPC after error@1450438787) -> (continuation of Deferred@839629016 after retry RPC after error@1276495801) -> (continuation of Deferred@683452746 after retry RPC after error@1635951414) -> (continuation of Deferred@236984998 after retry RPC after error@1818901716) -> (continuation of Deferred@567361258 after retry RPC after error@1430172405)))
06:34:18.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.588 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.588 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.588 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.628 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.650 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.650 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.650 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.651 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.731 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1829159778(state=PENDING, result=null, callback=(continuation of Deferred@1021567843 after retry RPC after error@25312396) -> (continuation of Deferred@464890734 after retry RPC after error@1089693353) -> (continuation of Deferred@1671982383 after retry RPC after error@1935559562) -> (continuation of Deferred@1158049876 after retry RPC after error@505118835) -> (continuation of Deferred@1048205194 after retry RPC after error@1073306068) -> (continuation of Deferred@1233849007 after retry RPC after error@189862901), errback=(continuation of Deferred@1021567843 after retry RPC after error@25312396) -> (continuation of Deferred@464890734 after retry RPC after error@1089693353) -> (continuation of Deferred@1671982383 after retry RPC after error@1935559562) -> (continuation of Deferred@1158049876 after retry RPC after error@505118835) -> (continuation of Deferred@1048205194 after retry RPC after error@1073306068) -> (continuation of Deferred@1233849007 after retry RPC after error@189862901)))
06:34:18.731 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1829159778(state=PENDING, result=null, callback=(continuation of Deferred@1021567843 after retry RPC after error@25312396) -> (continuation of Deferred@464890734 after retry RPC after error@1089693353) -> (continuation of Deferred@1671982383 after retry RPC after error@1935559562) -> (continuation of Deferred@1158049876 after retry RPC after error@505118835) -> (continuation of Deferred@1048205194 after retry RPC after error@1073306068) -> (continuation of Deferred@1233849007 after retry RPC after error@189862901), errback=(continuation of Deferred@1021567843 after retry RPC after error@25312396) -> (continuation of Deferred@464890734 after retry RPC after error@1089693353) -> (continuation of Deferred@1671982383 after retry RPC after error@1935559562) -> (continuation of Deferred@1158049876 after retry RPC after error@505118835) -> (continuation of Deferred@1048205194 after retry RPC after error@1073306068) -> (continuation of Deferred@1233849007 after retry RPC after error@189862901)))
06:34:18.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.750 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.750 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.750 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.751 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.788 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.828 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.828 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:18.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:18.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:18.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:18.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:18.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.009 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1560332777(state=PENDING, result=null, callback=(continuation of Deferred@2121465128 after retry RPC after error@553680043) -> (continuation of Deferred@2146391266 after retry RPC after error@620467575) -> (continuation of Deferred@1292482429 after retry RPC after error@1875803987) -> (continuation of Deferred@1787002380 after retry RPC after error@304811526) -> (continuation of Deferred@312078689 after retry RPC after error@246493318) -> (continuation of Deferred@2142456898 after retry RPC after error@891434636) -> (continuation of Deferred@1752030128 after retry RPC after error@1868555067), errback=(continuation of Deferred@2121465128 after retry RPC after error@553680043) -> (continuation of Deferred@2146391266 after retry RPC after error@620467575) -> (continuation of Deferred@1292482429 after retry RPC after error@1875803987) -> (continuation of Deferred@1787002380 after retry RPC after error@304811526) -> (continuation of Deferred@312078689 after retry RPC after error@246493318) -> (continuation of Deferred@2142456898 after retry RPC after error@891434636) -> (continuation of Deferred@1752030128 after retry RPC after error@1868555067)))
06:34:19.009 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:19.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.050 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.050 [WARN - kudu-nio-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.109 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.210 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.210 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@889974892(state=PENDING, result=null, callback=(continuation of Deferred@148947038 after retry RPC after error@1157340522) -> (continuation of Deferred@1860620926 after retry RPC after error@75814750) -> (continuation of Deferred@174589344 after retry RPC after error@308336970) -> (continuation of Deferred@1254984794 after retry RPC after error@1758563450) -> (continuation of Deferred@585028037 after retry RPC after error@912320093) -> (continuation of Deferred@275317655 after retry RPC after error@1958735804), errback=(continuation of Deferred@148947038 after retry RPC after error@1157340522) -> (continuation of Deferred@1860620926 after retry RPC after error@75814750) -> (continuation of Deferred@174589344 after retry RPC after error@308336970) -> (continuation of Deferred@1254984794 after retry RPC after error@1758563450) -> (continuation of Deferred@585028037 after retry RPC after error@912320093) -> (continuation of Deferred@275317655 after retry RPC after error@1958735804)))
06:34:19.211 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@889974892(state=PENDING, result=null, callback=(continuation of Deferred@148947038 after retry RPC after error@1157340522) -> (continuation of Deferred@1860620926 after retry RPC after error@75814750) -> (continuation of Deferred@174589344 after retry RPC after error@308336970) -> (continuation of Deferred@1254984794 after retry RPC after error@1758563450) -> (continuation of Deferred@585028037 after retry RPC after error@912320093) -> (continuation of Deferred@275317655 after retry RPC after error@1958735804), errback=(continuation of Deferred@148947038 after retry RPC after error@1157340522) -> (continuation of Deferred@1860620926 after retry RPC after error@75814750) -> (continuation of Deferred@174589344 after retry RPC after error@308336970) -> (continuation of Deferred@1254984794 after retry RPC after error@1758563450) -> (continuation of Deferred@585028037 after retry RPC after error@912320093) -> (continuation of Deferred@275317655 after retry RPC after error@1958735804)))
06:34:19.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.230 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.268 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.350 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.350 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.351 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.451 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [39ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1623603440(state=PENDING, result=null, callback=(continuation of Deferred@298931713 after retry RPC after error@759897724) -> (continuation of Deferred@1481441127 after retry RPC after error@152849702) -> (continuation of Deferred@308704090 after retry RPC after error@1730885524) -> (continuation of Deferred@1487100881 after retry RPC after error@859654302) -> (continuation of Deferred@316674304 after retry RPC after error@751375199) -> (continuation of Deferred@1853270212 after retry RPC after error@936797228) -> (continuation of Deferred@632032199 after retry RPC after error@1092331462), errback=(continuation of Deferred@298931713 after retry RPC after error@759897724) -> (continuation of Deferred@1481441127 after retry RPC after error@152849702) -> (continuation of Deferred@308704090 after retry RPC after error@1730885524) -> (continuation of Deferred@1487100881 after retry RPC after error@859654302) -> (continuation of Deferred@316674304 after retry RPC after error@751375199) -> (continuation of Deferred@1853270212 after retry RPC after error@936797228) -> (continuation of Deferred@632032199 after retry RPC after error@1092331462)))
06:34:19.451 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [39ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [139ms] refreshing cache from master, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [139ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [141ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [221ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1623603440(state=PENDING, result=null, callback=(continuation of Deferred@298931713 after retry RPC after error@759897724) -> (continuation of Deferred@1481441127 after retry RPC after error@152849702) -> (continuation of Deferred@308704090 after retry RPC after error@1730885524) -> (continuation of Deferred@1487100881 after retry RPC after error@859654302) -> (continuation of Deferred@316674304 after retry RPC after error@751375199) -> (continuation of Deferred@1853270212 after retry RPC after error@936797228) -> (continuation of Deferred@632032199 after retry RPC after error@1092331462), errback=(continuation of Deferred@298931713 after retry RPC after error@759897724) -> (continuation of Deferred@1481441127 after retry RPC after error@152849702) -> (continuation of Deferred@308704090 after retry RPC after error@1730885524) -> (continuation of Deferred@1487100881 after retry RPC after error@859654302) -> (continuation of Deferred@316674304 after retry RPC after error@751375199) -> (continuation of Deferred@1853270212 after retry RPC after error@936797228) -> (continuation of Deferred@632032199 after retry RPC after error@1092331462)))
06:34:19.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.510 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.528 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.528 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.671 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1381785192(state=PENDING, result=null, callback=(continuation of Deferred@197753054 after retry RPC after error@1632852794) -> (continuation of Deferred@1967526988 after retry RPC after error@1163504530) -> (continuation of Deferred@85341295 after retry RPC after error@2129077628) -> (continuation of Deferred@994486936 after retry RPC after error@860586841) -> (continuation of Deferred@130706451 after retry RPC after error@1086420307) -> (continuation of Deferred@1314719161 after retry RPC after error@568526432) -> (continuation of Deferred@1973247163 after retry RPC after error@1052624054), errback=(continuation of Deferred@197753054 after retry RPC after error@1632852794) -> (continuation of Deferred@1967526988 after retry RPC after error@1163504530) -> (continuation of Deferred@85341295 after retry RPC after error@2129077628) -> (continuation of Deferred@994486936 after retry RPC after error@860586841) -> (continuation of Deferred@130706451 after retry RPC after error@1086420307) -> (continuation of Deferred@1314719161 after retry RPC after error@568526432) -> (continuation of Deferred@1973247163 after retry RPC after error@1052624054)))
06:34:19.671 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1381785192(state=PENDING, result=null, callback=(continuation of Deferred@197753054 after retry RPC after error@1632852794) -> (continuation of Deferred@1967526988 after retry RPC after error@1163504530) -> (continuation of Deferred@85341295 after retry RPC after error@2129077628) -> (continuation of Deferred@994486936 after retry RPC after error@860586841) -> (continuation of Deferred@130706451 after retry RPC after error@1086420307) -> (continuation of Deferred@1314719161 after retry RPC after error@568526432) -> (continuation of Deferred@1973247163 after retry RPC after error@1052624054), errback=(continuation of Deferred@197753054 after retry RPC after error@1632852794) -> (continuation of Deferred@1967526988 after retry RPC after error@1163504530) -> (continuation of Deferred@85341295 after retry RPC after error@2129077628) -> (continuation of Deferred@994486936 after retry RPC after error@860586841) -> (continuation of Deferred@130706451 after retry RPC after error@1086420307) -> (continuation of Deferred@1314719161 after retry RPC after error@568526432) -> (continuation of Deferred@1973247163 after retry RPC after error@1052624054)))
06:34:19.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.689 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.729 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.750 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.768 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.948 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@308289875(state=PENDING, result=null, callback=(continuation of Deferred@909333493 after retry RPC after error@1710141551) -> (continuation of Deferred@2016250414 after retry RPC after error@1809073857) -> (continuation of Deferred@1205083317 after retry RPC after error@1462346216) -> (continuation of Deferred@1136733657 after retry RPC after error@183146798) -> (continuation of Deferred@1841044850 after retry RPC after error@2004932556) -> (continuation of Deferred@1038455552 after retry RPC after error@1670925947) -> (continuation of Deferred@1849854199 after retry RPC after error@520948804), errback=(continuation of Deferred@909333493 after retry RPC after error@1710141551) -> (continuation of Deferred@2016250414 after retry RPC after error@1809073857) -> (continuation of Deferred@1205083317 after retry RPC after error@1462346216) -> (continuation of Deferred@1136733657 after retry RPC after error@183146798) -> (continuation of Deferred@1841044850 after retry RPC after error@2004932556) -> (continuation of Deferred@1038455552 after retry RPC after error@1670925947) -> (continuation of Deferred@1849854199 after retry RPC after error@520948804)))
06:34:19.949 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:19.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.970 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.970 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.971 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:19.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:19.989 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:19.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:19.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:19.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.008 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.008 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.088 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.110 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.110 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.210 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.210194 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:20.214 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.214638 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '22852a0334cf48fbb5c4242fe8272a67', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:20.216 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.216054 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:20.229 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2030544039(state=PENDING, result=null, callback=(continuation of Deferred@1125932033 after retry RPC after error@1324915256) -> (continuation of Deferred@1628182010 after retry RPC after error@627750880) -> (continuation of Deferred@70716184 after retry RPC after error@236222716) -> (continuation of Deferred@255205121 after retry RPC after error@2131979440) -> (continuation of Deferred@1136707457 after retry RPC after error@1857983645) -> (continuation of Deferred@1364148316 after retry RPC after error@903729193) -> (continuation of Deferred@1894053340 after retry RPC after error@1784978721), errback=(continuation of Deferred@1125932033 after retry RPC after error@1324915256) -> (continuation of Deferred@1628182010 after retry RPC after error@627750880) -> (continuation of Deferred@70716184 after retry RPC after error@236222716) -> (continuation of Deferred@255205121 after retry RPC after error@2131979440) -> (continuation of Deferred@1136707457 after retry RPC after error@1857983645) -> (continuation of Deferred@1364148316 after retry RPC after error@903729193) -> (continuation of Deferred@1894053340 after retry RPC after error@1784978721)))
06:34:20.229 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:20.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:20.346063 25752 raft_consensus.cc:482] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election (detected failure of leader d573f1a12d4c4086bbf20f8e73481ad7)
06:34:20.346 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:20.346323 25752 raft_consensus.cc:504] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Starting pre-election with config: opid_index: -1 OBSOLETE_local: false peers { permanent_uuid: "ceef4bfa72a4468aad391f4034246263" member_type: VOTER last_known_addr { host: "127.23.161.254" port: 38015 } } peers { permanent_uuid: "eb210d252711436ab28b2ae64acb48a3" member_type: VOTER last_known_addr { host: "127.23.161.253" port: 45177 } } peers { permanent_uuid: "d573f1a12d4c4086bbf20f8e73481ad7" member_type: VOTER last_known_addr { host: "127.23.161.252" port: 42373 } }
06:34:20.347 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:20.347034 25752 leader_election.cc:288] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Requested pre-vote from peers ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015), d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373)
06:34:20.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.348 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.348358 25336 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111) [suppressed 1 similar messages]
06:34:20.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.350311 25336 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer d573f1a12d4c4086bbf20f8e73481ad7 (127.23.161.252:42373): Network error: Client connection negotiation failed: client connection to 127.23.161.252:42373: connect: Connection refused (error 111)
06:34:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:20.350484 25333 leader_election.cc:334] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: RPC error from VoteRequest() call to peer ceef4bfa72a4468aad391f4034246263 (127.23.161.254:38015): Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111)
06:34:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:20.350584 25333 leader_election.cc:302] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [CANDIDATE]: Term 5 pre-election: Election decided. Result: candidate lost. Election summary: received 3 responses out of 3 voters: 1 yes votes; 2 no votes. yes voters: eb210d252711436ab28b2ae64acb48a3; no voters: ceef4bfa72a4468aad391f4034246263, d573f1a12d4c4086bbf20f8e73481ad7
06:34:20.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.350 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:20.350795 25752 raft_consensus.cc:2726] T 00000000000000000000000000000000 P eb210d252711436ab28b2ae64acb48a3 [term 4 FOLLOWER]: Leader pre-election lost for term 5. Reason: could not achieve majority
06:34:20.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.390 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.390 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.390 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.410 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@259694011(state=PENDING, result=null, callback=(continuation of Deferred@796302512 after retry RPC after error@1426004531) -> (continuation of Deferred@153047080 after retry RPC after error@1381209077) -> (continuation of Deferred@1239094803 after retry RPC after error@1708766105) -> (continuation of Deferred@372519194 after retry RPC after error@1479984146) -> (continuation of Deferred@1238121097 after retry RPC after error@1629556050) -> (continuation of Deferred@1445612685 after retry RPC after error@944899086) -> (continuation of Deferred@274885119 after retry RPC after error@2042674492), errback=(continuation of Deferred@796302512 after retry RPC after error@1426004531) -> (continuation of Deferred@153047080 after retry RPC after error@1381209077) -> (continuation of Deferred@1239094803 after retry RPC after error@1708766105) -> (continuation of Deferred@372519194 after retry RPC after error@1479984146) -> (continuation of Deferred@1238121097 after retry RPC after error@1629556050) -> (continuation of Deferred@1445612685 after retry RPC after error@944899086) -> (continuation of Deferred@274885119 after retry RPC after error@2042674492)))
06:34:20.411 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@259694011(state=PENDING, result=null, callback=(continuation of Deferred@796302512 after retry RPC after error@1426004531) -> (continuation of Deferred@153047080 after retry RPC after error@1381209077) -> (continuation of Deferred@1239094803 after retry RPC after error@1708766105) -> (continuation of Deferred@372519194 after retry RPC after error@1479984146) -> (continuation of Deferred@1238121097 after retry RPC after error@1629556050) -> (continuation of Deferred@1445612685 after retry RPC after error@944899086) -> (continuation of Deferred@274885119 after retry RPC after error@2042674492), errback=(continuation of Deferred@796302512 after retry RPC after error@1426004531) -> (continuation of Deferred@153047080 after retry RPC after error@1381209077) -> (continuation of Deferred@1239094803 after retry RPC after error@1708766105) -> (continuation of Deferred@372519194 after retry RPC after error@1479984146) -> (continuation of Deferred@1238121097 after retry RPC after error@1629556050) -> (continuation of Deferred@1445612685 after retry RPC after error@944899086) -> (continuation of Deferred@274885119 after retry RPC after error@2042674492)))
06:34:20.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.489 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.528 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.529 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.570 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.570 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.571 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.589 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.590 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.591 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@134492688(state=PENDING, result=null, callback=(continuation of Deferred@1259460333 after retry RPC after error@1353396378) -> (continuation of Deferred@760779064 after retry RPC after error@73773617) -> (continuation of Deferred@1090960110 after retry RPC after error@1218477492) -> (continuation of Deferred@563668169 after retry RPC after error@147140337) -> (continuation of Deferred@2086675661 after retry RPC after error@937863976) -> (continuation of Deferred@1371455409 after retry RPC after error@1890431441) -> (continuation of Deferred@219592005 after retry RPC after error@262926372), errback=(continuation of Deferred@1259460333 after retry RPC after error@1353396378) -> (continuation of Deferred@760779064 after retry RPC after error@73773617) -> (continuation of Deferred@1090960110 after retry RPC after error@1218477492) -> (continuation of Deferred@563668169 after retry RPC after error@147140337) -> (continuation of Deferred@2086675661 after retry RPC after error@937863976) -> (continuation of Deferred@1371455409 after retry RPC after error@1890431441) -> (continuation of Deferred@219592005 after retry RPC after error@262926372)))
06:34:20.591 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@134492688(state=PENDING, result=null, callback=(continuation of Deferred@1259460333 after retry RPC after error@1353396378) -> (continuation of Deferred@760779064 after retry RPC after error@73773617) -> (continuation of Deferred@1090960110 after retry RPC after error@1218477492) -> (continuation of Deferred@563668169 after retry RPC after error@147140337) -> (continuation of Deferred@2086675661 after retry RPC after error@937863976) -> (continuation of Deferred@1371455409 after retry RPC after error@1890431441) -> (continuation of Deferred@219592005 after retry RPC after error@262926372), errback=(continuation of Deferred@1259460333 after retry RPC after error@1353396378) -> (continuation of Deferred@760779064 after retry RPC after error@73773617) -> (continuation of Deferred@1090960110 after retry RPC after error@1218477492) -> (continuation of Deferred@563668169 after retry RPC after error@147140337) -> (continuation of Deferred@2086675661 after retry RPC after error@937863976) -> (continuation of Deferred@1371455409 after retry RPC after error@1890431441) -> (continuation of Deferred@219592005 after retry RPC after error@262926372)))
06:34:20.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.608 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.608 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.630 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.631 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.668 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.668 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.728 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.728 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.729 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.749 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.751 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.831 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1100457287(state=PENDING, result=null, callback=(continuation of Deferred@1095659200 after retry RPC after error@137877626) -> (continuation of Deferred@330127517 after retry RPC after error@1877764658) -> (continuation of Deferred@1575367009 after retry RPC after error@1797572271) -> (continuation of Deferred@313791206 after retry RPC after error@503662456) -> (continuation of Deferred@1912864992 after retry RPC after error@1062777265) -> (continuation of Deferred@222930846 after retry RPC after error@969084872) -> (continuation of Deferred@1796344592 after retry RPC after error@297663226) -> (continuation of Deferred@1359530500 after retry RPC after error@1888760482), errback=(continuation of Deferred@1095659200 after retry RPC after error@137877626) -> (continuation of Deferred@330127517 after retry RPC after error@1877764658) -> (continuation of Deferred@1575367009 after retry RPC after error@1797572271) -> (continuation of Deferred@313791206 after retry RPC after error@503662456) -> (continuation of Deferred@1912864992 after retry RPC after error@1062777265) -> (continuation of Deferred@222930846 after retry RPC after error@969084872) -> (continuation of Deferred@1796344592 after retry RPC after error@297663226) -> (continuation of Deferred@1359530500 after retry RPC after error@1888760482)))
06:34:20.831 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [143ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1100457287(state=PENDING, result=null, callback=(continuation of Deferred@1095659200 after retry RPC after error@137877626) -> (continuation of Deferred@330127517 after retry RPC after error@1877764658) -> (continuation of Deferred@1575367009 after retry RPC after error@1797572271) -> (continuation of Deferred@313791206 after retry RPC after error@503662456) -> (continuation of Deferred@1912864992 after retry RPC after error@1062777265) -> (continuation of Deferred@222930846 after retry RPC after error@969084872) -> (continuation of Deferred@1796344592 after retry RPC after error@297663226) -> (continuation of Deferred@1359530500 after retry RPC after error@1888760482), errback=(continuation of Deferred@1095659200 after retry RPC after error@137877626) -> (continuation of Deferred@330127517 after retry RPC after error@1877764658) -> (continuation of Deferred@1575367009 after retry RPC after error@1797572271) -> (continuation of Deferred@313791206 after retry RPC after error@503662456) -> (continuation of Deferred@1912864992 after retry RPC after error@1062777265) -> (continuation of Deferred@222930846 after retry RPC after error@969084872) -> (continuation of Deferred@1796344592 after retry RPC after error@297663226) -> (continuation of Deferred@1359530500 after retry RPC after error@1888760482)))
06:34:20.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.850 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.850 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.870 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.870 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:20.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:20.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:20.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:20.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:20.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.108 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [159ms] refreshing cache from master, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [159ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [161ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [239ms] refreshing cache from master, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [239ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [241ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@689703938(state=PENDING, result=null, callback=(continuation of Deferred@947867189 after retry RPC after error@1095588836) -> (continuation of Deferred@2102883760 after retry RPC after error@887233028) -> (continuation of Deferred@1991173195 after retry RPC after error@1570181889) -> (continuation of Deferred@464197622 after retry RPC after error@1060965565) -> (continuation of Deferred@1043348397 after retry RPC after error@1541027953) -> (continuation of Deferred@403538011 after retry RPC after error@1333931779) -> (continuation of Deferred@1884329997 after retry RPC after error@2059616198) -> (continuation of Deferred@238581661 after retry RPC after error@1352448878), errback=(continuation of Deferred@947867189 after retry RPC after error@1095588836) -> (continuation of Deferred@2102883760 after retry RPC after error@887233028) -> (continuation of Deferred@1991173195 after retry RPC after error@1570181889) -> (continuation of Deferred@464197622 after retry RPC after error@1060965565) -> (continuation of Deferred@1043348397 after retry RPC after error@1541027953) -> (continuation of Deferred@403538011 after retry RPC after error@1333931779) -> (continuation of Deferred@1884329997 after retry RPC after error@2059616198) -> (continuation of Deferred@238581661 after retry RPC after error@1352448878)))
06:34:21.109 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:21.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.130 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.130 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.270 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.290 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1081575699(state=PENDING, result=null, callback=(continuation of Deferred@2070717767 after retry RPC after error@1848935768) -> (continuation of Deferred@2056922509 after retry RPC after error@106501340) -> (continuation of Deferred@411160525 after retry RPC after error@1381337024) -> (continuation of Deferred@2043180618 after retry RPC after error@939094619) -> (continuation of Deferred@563100878 after retry RPC after error@1594217659) -> (continuation of Deferred@1699058487 after retry RPC after error@424824430) -> (continuation of Deferred@964978630 after retry RPC after error@1221971009), errback=(continuation of Deferred@2070717767 after retry RPC after error@1848935768) -> (continuation of Deferred@2056922509 after retry RPC after error@106501340) -> (continuation of Deferred@411160525 after retry RPC after error@1381337024) -> (continuation of Deferred@2043180618 after retry RPC after error@939094619) -> (continuation of Deferred@563100878 after retry RPC after error@1594217659) -> (continuation of Deferred@1699058487 after retry RPC after error@424824430) -> (continuation of Deferred@964978630 after retry RPC after error@1221971009)))
06:34:21.291 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [103ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1081575699(state=PENDING, result=null, callback=(continuation of Deferred@2070717767 after retry RPC after error@1848935768) -> (continuation of Deferred@2056922509 after retry RPC after error@106501340) -> (continuation of Deferred@411160525 after retry RPC after error@1381337024) -> (continuation of Deferred@2043180618 after retry RPC after error@939094619) -> (continuation of Deferred@563100878 after retry RPC after error@1594217659) -> (continuation of Deferred@1699058487 after retry RPC after error@424824430) -> (continuation of Deferred@964978630 after retry RPC after error@1221971009), errback=(continuation of Deferred@2070717767 after retry RPC after error@1848935768) -> (continuation of Deferred@2056922509 after retry RPC after error@106501340) -> (continuation of Deferred@411160525 after retry RPC after error@1381337024) -> (continuation of Deferred@2043180618 after retry RPC after error@939094619) -> (continuation of Deferred@563100878 after retry RPC after error@1594217659) -> (continuation of Deferred@1699058487 after retry RPC after error@424824430) -> (continuation of Deferred@964978630 after retry RPC after error@1221971009)))
06:34:21.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.309 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.309 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.330 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.368 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1255144042(state=PENDING, result=null, callback=(continuation of Deferred@1788567674 after retry RPC after error@1697836542) -> (continuation of Deferred@666487254 after retry RPC after error@303070177) -> (continuation of Deferred@599343465 after retry RPC after error@1368790525) -> (continuation of Deferred@948470112 after retry RPC after error@693517027) -> (continuation of Deferred@1083016361 after retry RPC after error@567625912) -> (continuation of Deferred@1563053667 after retry RPC after error@1161029127), errback=(continuation of Deferred@1788567674 after retry RPC after error@1697836542) -> (continuation of Deferred@666487254 after retry RPC after error@303070177) -> (continuation of Deferred@599343465 after retry RPC after error@1368790525) -> (continuation of Deferred@948470112 after retry RPC after error@693517027) -> (continuation of Deferred@1083016361 after retry RPC after error@567625912) -> (continuation of Deferred@1563053667 after retry RPC after error@1161029127)))
06:34:21.511 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1255144042(state=PENDING, result=null, callback=(continuation of Deferred@1788567674 after retry RPC after error@1697836542) -> (continuation of Deferred@666487254 after retry RPC after error@303070177) -> (continuation of Deferred@599343465 after retry RPC after error@1368790525) -> (continuation of Deferred@948470112 after retry RPC after error@693517027) -> (continuation of Deferred@1083016361 after retry RPC after error@567625912) -> (continuation of Deferred@1563053667 after retry RPC after error@1161029127), errback=(continuation of Deferred@1788567674 after retry RPC after error@1697836542) -> (continuation of Deferred@666487254 after retry RPC after error@303070177) -> (continuation of Deferred@599343465 after retry RPC after error@1368790525) -> (continuation of Deferred@948470112 after retry RPC after error@693517027) -> (continuation of Deferred@1083016361 after retry RPC after error@567625912) -> (continuation of Deferred@1563053667 after retry RPC after error@1161029127)))
06:34:21.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.550 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.550 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.588 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.668 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.788 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1447283216(state=PENDING, result=null, callback=(continuation of Deferred@479189605 after retry RPC after error@2079958400) -> (continuation of Deferred@2070836223 after retry RPC after error@867509704) -> (continuation of Deferred@1368104233 after retry RPC after error@1743320893) -> (continuation of Deferred@1863418292 after retry RPC after error@1972400891) -> (continuation of Deferred@2021641141 after retry RPC after error@1477352088) -> (continuation of Deferred@1005698060 after retry RPC after error@1291851299) -> (continuation of Deferred@1260670549 after retry RPC after error@1733482483), errback=(continuation of Deferred@479189605 after retry RPC after error@2079958400) -> (continuation of Deferred@2070836223 after retry RPC after error@867509704) -> (continuation of Deferred@1368104233 after retry RPC after error@1743320893) -> (continuation of Deferred@1863418292 after retry RPC after error@1972400891) -> (continuation of Deferred@2021641141 after retry RPC after error@1477352088) -> (continuation of Deferred@1005698060 after retry RPC after error@1291851299) -> (continuation of Deferred@1260670549 after retry RPC after error@1733482483)))
06:34:21.789 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:21.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.829 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.831 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.848 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.848 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.870 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.910 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.910 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:21.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:21.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:21.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:21.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:21.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.069 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2125441912(state=PENDING, result=null, callback=(continuation of Deferred@1638992434 after retry RPC after error@215312090) -> (continuation of Deferred@522059225 after retry RPC after error@485122345) -> (continuation of Deferred@1977817116 after retry RPC after error@1386697676) -> (continuation of Deferred@581214068 after retry RPC after error@1829257504) -> (continuation of Deferred@13010789 after retry RPC after error@1754762626) -> (continuation of Deferred@1850663131 after retry RPC after error@1105247603) -> (continuation of Deferred@1965680517 after retry RPC after error@554078015) -> (continuation of Deferred@953674748 after retry RPC after error@2090386096), errback=(continuation of Deferred@1638992434 after retry RPC after error@215312090) -> (continuation of Deferred@522059225 after retry RPC after error@485122345) -> (continuation of Deferred@1977817116 after retry RPC after error@1386697676) -> (continuation of Deferred@581214068 after retry RPC after error@1829257504) -> (continuation of Deferred@13010789 after retry RPC after error@1754762626) -> (continuation of Deferred@1850663131 after retry RPC after error@1105247603) -> (continuation of Deferred@1965680517 after retry RPC after error@554078015) -> (continuation of Deferred@953674748 after retry RPC after error@2090386096)))
06:34:22.069 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:22.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.088 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.089 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.089 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.089 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.090 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.109 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.109 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.110 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.110 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.128 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.150 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.150 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.192 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.192 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.193 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.193 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.193 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.193 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.193 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.249 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.250 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.250 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.251 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [105ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@289454600(state=PENDING, result=null, callback=(continuation of Deferred@1699419194 after retry RPC after error@1900800064) -> (continuation of Deferred@928265664 after retry RPC after error@592464344) -> (continuation of Deferred@503800652 after retry RPC after error@815829284) -> (continuation of Deferred@642537669 after retry RPC after error@133417821) -> (continuation of Deferred@575391212 after retry RPC after error@374757680) -> (continuation of Deferred@1385734479 after retry RPC after error@1260921186), errback=(continuation of Deferred@1699419194 after retry RPC after error@1900800064) -> (continuation of Deferred@928265664 after retry RPC after error@592464344) -> (continuation of Deferred@503800652 after retry RPC after error@815829284) -> (continuation of Deferred@642537669 after retry RPC after error@133417821) -> (continuation of Deferred@575391212 after retry RPC after error@374757680) -> (continuation of Deferred@1385734479 after retry RPC after error@1260921186)))
06:34:22.252 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=163), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [105ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@289454600(state=PENDING, result=null, callback=(continuation of Deferred@1699419194 after retry RPC after error@1900800064) -> (continuation of Deferred@928265664 after retry RPC after error@592464344) -> (continuation of Deferred@503800652 after retry RPC after error@815829284) -> (continuation of Deferred@642537669 after retry RPC after error@133417821) -> (continuation of Deferred@575391212 after retry RPC after error@374757680) -> (continuation of Deferred@1385734479 after retry RPC after error@1260921186), errback=(continuation of Deferred@1699419194 after retry RPC after error@1900800064) -> (continuation of Deferred@928265664 after retry RPC after error@592464344) -> (continuation of Deferred@503800652 after retry RPC after error@815829284) -> (continuation of Deferred@642537669 after retry RPC after error@133417821) -> (continuation of Deferred@575391212 after retry RPC after error@374757680) -> (continuation of Deferred@1385734479 after retry RPC after error@1260921186)))
06:34:22.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.269 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.270 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.271 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.288 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.290 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.308 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.308 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.308 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.309 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.309 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.309 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.309 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.310 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:22.327297 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:22.327 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:22.327546 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:22.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.430 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.431 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2061784091(state=PENDING, result=null, callback=(continuation of Deferred@1332112535 after retry RPC after error@1616782703) -> (continuation of Deferred@153017377 after retry RPC after error@1359091093) -> (continuation of Deferred@763152886 after retry RPC after error@1016652907) -> (continuation of Deferred@1134090253 after retry RPC after error@41403222) -> (continuation of Deferred@1234143639 after retry RPC after error@844076913) -> (continuation of Deferred@932342149 after retry RPC after error@188969298), errback=(continuation of Deferred@1332112535 after retry RPC after error@1616782703) -> (continuation of Deferred@153017377 after retry RPC after error@1359091093) -> (continuation of Deferred@763152886 after retry RPC after error@1016652907) -> (continuation of Deferred@1134090253 after retry RPC after error@41403222) -> (continuation of Deferred@1234143639 after retry RPC after error@844076913) -> (continuation of Deferred@932342149 after retry RPC after error@188969298)))
06:34:22.431 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=162), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [23ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@2061784091(state=PENDING, result=null, callback=(continuation of Deferred@1332112535 after retry RPC after error@1616782703) -> (continuation of Deferred@153017377 after retry RPC after error@1359091093) -> (continuation of Deferred@763152886 after retry RPC after error@1016652907) -> (continuation of Deferred@1134090253 after retry RPC after error@41403222) -> (continuation of Deferred@1234143639 after retry RPC after error@844076913) -> (continuation of Deferred@932342149 after retry RPC after error@188969298), errback=(continuation of Deferred@1332112535 after retry RPC after error@1616782703) -> (continuation of Deferred@153017377 after retry RPC after error@1359091093) -> (continuation of Deferred@763152886 after retry RPC after error@1016652907) -> (continuation of Deferred@1134090253 after retry RPC after error@41403222) -> (continuation of Deferred@1234143639 after retry RPC after error@844076913) -> (continuation of Deferred@932342149 after retry RPC after error@188969298)))
06:34:22.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.631 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1845002171(state=PENDING, result=null, callback=(continuation of Deferred@1494027197 after retry RPC after error@908730913) -> (continuation of Deferred@2070190789 after retry RPC after error@1439643573) -> (continuation of Deferred@1680776786 after retry RPC after error@66225116) -> (continuation of Deferred@1545573333 after retry RPC after error@1891722620) -> (continuation of Deferred@1611512120 after retry RPC after error@2012625542) -> (continuation of Deferred@1465355926 after retry RPC after error@751233925), errback=(continuation of Deferred@1494027197 after retry RPC after error@908730913) -> (continuation of Deferred@2070190789 after retry RPC after error@1439643573) -> (continuation of Deferred@1680776786 after retry RPC after error@66225116) -> (continuation of Deferred@1545573333 after retry RPC after error@1891722620) -> (continuation of Deferred@1611512120 after retry RPC after error@2012625542) -> (continuation of Deferred@1465355926 after retry RPC after error@751233925)))
06:34:22.631 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1845002171(state=PENDING, result=null, callback=(continuation of Deferred@1494027197 after retry RPC after error@908730913) -> (continuation of Deferred@2070190789 after retry RPC after error@1439643573) -> (continuation of Deferred@1680776786 after retry RPC after error@66225116) -> (continuation of Deferred@1545573333 after retry RPC after error@1891722620) -> (continuation of Deferred@1611512120 after retry RPC after error@2012625542) -> (continuation of Deferred@1465355926 after retry RPC after error@751233925), errback=(continuation of Deferred@1494027197 after retry RPC after error@908730913) -> (continuation of Deferred@2070190789 after retry RPC after error@1439643573) -> (continuation of Deferred@1680776786 after retry RPC after error@66225116) -> (continuation of Deferred@1545573333 after retry RPC after error@1891722620) -> (continuation of Deferred@1611512120 after retry RPC after error@2012625542) -> (continuation of Deferred@1465355926 after retry RPC after error@751233925)))
06:34:22.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.730 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.891 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@326031554(state=PENDING, result=null, callback=(continuation of Deferred@1277986413 after retry RPC after error@2051520244) -> (continuation of Deferred@416354301 after retry RPC after error@2107407597) -> (continuation of Deferred@1428914909 after retry RPC after error@312245914) -> (continuation of Deferred@35792339 after retry RPC after error@482826862) -> (continuation of Deferred@359861825 after retry RPC after error@1684724443) -> (continuation of Deferred@2012575544 after retry RPC after error@1372014466) -> (continuation of Deferred@1556826877 after retry RPC after error@1858469821), errback=(continuation of Deferred@1277986413 after retry RPC after error@2051520244) -> (continuation of Deferred@416354301 after retry RPC after error@2107407597) -> (continuation of Deferred@1428914909 after retry RPC after error@312245914) -> (continuation of Deferred@35792339 after retry RPC after error@482826862) -> (continuation of Deferred@359861825 after retry RPC after error@1684724443) -> (continuation of Deferred@2012575544 after retry RPC after error@1372014466) -> (continuation of Deferred@1556826877 after retry RPC after error@1858469821)))
06:34:22.891 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@326031554(state=PENDING, result=null, callback=(continuation of Deferred@1277986413 after retry RPC after error@2051520244) -> (continuation of Deferred@416354301 after retry RPC after error@2107407597) -> (continuation of Deferred@1428914909 after retry RPC after error@312245914) -> (continuation of Deferred@35792339 after retry RPC after error@482826862) -> (continuation of Deferred@359861825 after retry RPC after error@1684724443) -> (continuation of Deferred@2012575544 after retry RPC after error@1372014466) -> (continuation of Deferred@1556826877 after retry RPC after error@1858469821), errback=(continuation of Deferred@1277986413 after retry RPC after error@2051520244) -> (continuation of Deferred@416354301 after retry RPC after error@2107407597) -> (continuation of Deferred@1428914909 after retry RPC after error@312245914) -> (continuation of Deferred@35792339 after retry RPC after error@482826862) -> (continuation of Deferred@359861825 after retry RPC after error@1684724443) -> (continuation of Deferred@2012575544 after retry RPC after error@1372014466) -> (continuation of Deferred@1556826877 after retry RPC after error@1858469821)))
06:34:22.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:22.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:22.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:22.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:22.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:22.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.048 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.049 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.049 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.049 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.049 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.049 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.049 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.049 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.050 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.068 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.150 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.151 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.151 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1078531727(state=PENDING, result=null, callback=(continuation of Deferred@1942948391 after retry RPC after error@143796597) -> (continuation of Deferred@911827023 after retry RPC after error@1051522405) -> (continuation of Deferred@901340837 after retry RPC after error@746131686) -> (continuation of Deferred@1373587006 after retry RPC after error@142710461) -> (continuation of Deferred@2008198526 after retry RPC after error@321420622) -> (continuation of Deferred@946677667 after retry RPC after error@1012759149) -> (continuation of Deferred@1583802821 after retry RPC after error@250883402) -> (continuation of Deferred@1601520392 after retry RPC after error@145540249), errback=(continuation of Deferred@1942948391 after retry RPC after error@143796597) -> (continuation of Deferred@911827023 after retry RPC after error@1051522405) -> (continuation of Deferred@901340837 after retry RPC after error@746131686) -> (continuation of Deferred@1373587006 after retry RPC after error@142710461) -> (continuation of Deferred@2008198526 after retry RPC after error@321420622) -> (continuation of Deferred@946677667 after retry RPC after error@1012759149) -> (continuation of Deferred@1583802821 after retry RPC after error@250883402) -> (continuation of Deferred@1601520392 after retry RPC after error@145540249)))
06:34:23.152 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [243ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [243ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1078531727(state=PENDING, result=null, callback=(continuation of Deferred@1942948391 after retry RPC after error@143796597) -> (continuation of Deferred@911827023 after retry RPC after error@1051522405) -> (continuation of Deferred@901340837 after retry RPC after error@746131686) -> (continuation of Deferred@1373587006 after retry RPC after error@142710461) -> (continuation of Deferred@2008198526 after retry RPC after error@321420622) -> (continuation of Deferred@946677667 after retry RPC after error@1012759149) -> (continuation of Deferred@1583802821 after retry RPC after error@250883402) -> (continuation of Deferred@1601520392 after retry RPC after error@145540249), errback=(continuation of Deferred@1942948391 after retry RPC after error@143796597) -> (continuation of Deferred@911827023 after retry RPC after error@1051522405) -> (continuation of Deferred@901340837 after retry RPC after error@746131686) -> (continuation of Deferred@1373587006 after retry RPC after error@142710461) -> (continuation of Deferred@2008198526 after retry RPC after error@321420622) -> (continuation of Deferred@946677667 after retry RPC after error@1012759149) -> (continuation of Deferred@1583802821 after retry RPC after error@250883402) -> (continuation of Deferred@1601520392 after retry RPC after error@145540249)))
06:34:23.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.208 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.248 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.248 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.248 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.249 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.289 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.291 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.349 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.350 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.351 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@167046493(state=PENDING, result=null, callback=(continuation of Deferred@1141384160 after retry RPC after error@2065978680) -> (continuation of Deferred@1460020865 after retry RPC after error@2042013497) -> (continuation of Deferred@2091158732 after retry RPC after error@628132853) -> (continuation of Deferred@1134855156 after retry RPC after error@258436054) -> (continuation of Deferred@1467827226 after retry RPC after error@1987269155) -> (continuation of Deferred@296735802 after retry RPC after error@631383042), errback=(continuation of Deferred@1141384160 after retry RPC after error@2065978680) -> (continuation of Deferred@1460020865 after retry RPC after error@2042013497) -> (continuation of Deferred@2091158732 after retry RPC after error@628132853) -> (continuation of Deferred@1134855156 after retry RPC after error@258436054) -> (continuation of Deferred@1467827226 after retry RPC after error@1987269155) -> (continuation of Deferred@296735802 after retry RPC after error@631383042)))
06:34:23.351 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=182), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@167046493(state=PENDING, result=null, callback=(continuation of Deferred@1141384160 after retry RPC after error@2065978680) -> (continuation of Deferred@1460020865 after retry RPC after error@2042013497) -> (continuation of Deferred@2091158732 after retry RPC after error@628132853) -> (continuation of Deferred@1134855156 after retry RPC after error@258436054) -> (continuation of Deferred@1467827226 after retry RPC after error@1987269155) -> (continuation of Deferred@296735802 after retry RPC after error@631383042), errback=(continuation of Deferred@1141384160 after retry RPC after error@2065978680) -> (continuation of Deferred@1460020865 after retry RPC after error@2042013497) -> (continuation of Deferred@2091158732 after retry RPC after error@628132853) -> (continuation of Deferred@1134855156 after retry RPC after error@258436054) -> (continuation of Deferred@1467827226 after retry RPC after error@1987269155) -> (continuation of Deferred@296735802 after retry RPC after error@631383042)))
06:34:23.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.571 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1368277457(state=PENDING, result=null, callback=(continuation of Deferred@360613910 after retry RPC after error@251839667) -> (continuation of Deferred@1108960601 after retry RPC after error@1631204354) -> (continuation of Deferred@15023627 after retry RPC after error@425416184) -> (continuation of Deferred@274843687 after retry RPC after error@541815821) -> (continuation of Deferred@167187305 after retry RPC after error@1254126394) -> (continuation of Deferred@1425980085 after retry RPC after error@804149051), errback=(continuation of Deferred@360613910 after retry RPC after error@251839667) -> (continuation of Deferred@1108960601 after retry RPC after error@1631204354) -> (continuation of Deferred@15023627 after retry RPC after error@425416184) -> (continuation of Deferred@274843687 after retry RPC after error@541815821) -> (continuation of Deferred@167187305 after retry RPC after error@1254126394) -> (continuation of Deferred@1425980085 after retry RPC after error@804149051)))
06:34:23.571 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] refreshing cache from master, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [23ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1368277457(state=PENDING, result=null, callback=(continuation of Deferred@360613910 after retry RPC after error@251839667) -> (continuation of Deferred@1108960601 after retry RPC after error@1631204354) -> (continuation of Deferred@15023627 after retry RPC after error@425416184) -> (continuation of Deferred@274843687 after retry RPC after error@541815821) -> (continuation of Deferred@167187305 after retry RPC after error@1254126394) -> (continuation of Deferred@1425980085 after retry RPC after error@804149051), errback=(continuation of Deferred@360613910 after retry RPC after error@251839667) -> (continuation of Deferred@1108960601 after retry RPC after error@1631204354) -> (continuation of Deferred@15023627 after retry RPC after error@425416184) -> (continuation of Deferred@274843687 after retry RPC after error@541815821) -> (continuation of Deferred@167187305 after retry RPC after error@1254126394) -> (continuation of Deferred@1425980085 after retry RPC after error@804149051)))
06:34:23.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.648 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.648 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.648 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.648 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.649 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.688 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.688 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.689 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.730 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@510933234(state=PENDING, result=null, callback=(continuation of Deferred@2097777493 after retry RPC after error@1011988602) -> (continuation of Deferred@2065420891 after retry RPC after error@1150797410) -> (continuation of Deferred@1979039366 after retry RPC after error@1146505770) -> (continuation of Deferred@1635434993 after retry RPC after error@455389034) -> (continuation of Deferred@1140663046 after retry RPC after error@889988681) -> (continuation of Deferred@38673488 after retry RPC after error@1009761420), errback=(continuation of Deferred@2097777493 after retry RPC after error@1011988602) -> (continuation of Deferred@2065420891 after retry RPC after error@1150797410) -> (continuation of Deferred@1979039366 after retry RPC after error@1146505770) -> (continuation of Deferred@1635434993 after retry RPC after error@455389034) -> (continuation of Deferred@1140663046 after retry RPC after error@889988681) -> (continuation of Deferred@38673488 after retry RPC after error@1009761420)))
06:34:23.731 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@510933234(state=PENDING, result=null, callback=(continuation of Deferred@2097777493 after retry RPC after error@1011988602) -> (continuation of Deferred@2065420891 after retry RPC after error@1150797410) -> (continuation of Deferred@1979039366 after retry RPC after error@1146505770) -> (continuation of Deferred@1635434993 after retry RPC after error@455389034) -> (continuation of Deferred@1140663046 after retry RPC after error@889988681) -> (continuation of Deferred@38673488 after retry RPC after error@1009761420), errback=(continuation of Deferred@2097777493 after retry RPC after error@1011988602) -> (continuation of Deferred@2065420891 after retry RPC after error@1150797410) -> (continuation of Deferred@1979039366 after retry RPC after error@1146505770) -> (continuation of Deferred@1635434993 after retry RPC after error@455389034) -> (continuation of Deferred@1140663046 after retry RPC after error@889988681) -> (continuation of Deferred@38673488 after retry RPC after error@1009761420)))
06:34:23.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.748 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.788 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.788 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.789 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.809 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.890 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@458826156(state=PENDING, result=null, callback=(continuation of Deferred@809210986 after retry RPC after error@432261283) -> (continuation of Deferred@634843808 after retry RPC after error@2010182647) -> (continuation of Deferred@455389603 after retry RPC after error@480638132) -> (continuation of Deferred@250893938 after retry RPC after error@2127250192) -> (continuation of Deferred@1088183491 after retry RPC after error@1936654392) -> (continuation of Deferred@1594138593 after retry RPC after error@2071172306), errback=(continuation of Deferred@809210986 after retry RPC after error@432261283) -> (continuation of Deferred@634843808 after retry RPC after error@2010182647) -> (continuation of Deferred@455389603 after retry RPC after error@480638132) -> (continuation of Deferred@250893938 after retry RPC after error@2127250192) -> (continuation of Deferred@1088183491 after retry RPC after error@1936654392) -> (continuation of Deferred@1594138593 after retry RPC after error@2071172306)))
06:34:23.891 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=142), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@458826156(state=PENDING, result=null, callback=(continuation of Deferred@809210986 after retry RPC after error@432261283) -> (continuation of Deferred@634843808 after retry RPC after error@2010182647) -> (continuation of Deferred@455389603 after retry RPC after error@480638132) -> (continuation of Deferred@250893938 after retry RPC after error@2127250192) -> (continuation of Deferred@1088183491 after retry RPC after error@1936654392) -> (continuation of Deferred@1594138593 after retry RPC after error@2071172306), errback=(continuation of Deferred@809210986 after retry RPC after error@432261283) -> (continuation of Deferred@634843808 after retry RPC after error@2010182647) -> (continuation of Deferred@455389603 after retry RPC after error@480638132) -> (continuation of Deferred@250893938 after retry RPC after error@2127250192) -> (continuation of Deferred@1088183491 after retry RPC after error@1936654392) -> (continuation of Deferred@1594138593 after retry RPC after error@2071172306)))
06:34:23.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.948 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:23.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:23.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:23.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.988 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:23.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:23.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.089 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.089 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.089 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.089 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.090 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.090 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.090 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.091 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.128 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.129 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.129 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.129 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.129 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.129 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.129 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.129 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.130 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.168 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [123ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [181ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [183ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@170493343(state=PENDING, result=null, callback=(continuation of Deferred@457251722 after retry RPC after error@4807576) -> (continuation of Deferred@1509483278 after retry RPC after error@532381907) -> (continuation of Deferred@204817893 after retry RPC after error@1181375072) -> (continuation of Deferred@1486102832 after retry RPC after error@459968489) -> (continuation of Deferred@1029110540 after retry RPC after error@704323484) -> (continuation of Deferred@1352607282 after retry RPC after error@533065231) -> (continuation of Deferred@365029472 after retry RPC after error@1681330875) -> (continuation of Deferred@1303385785 after retry RPC after error@2109025430), errback=(continuation of Deferred@457251722 after retry RPC after error@4807576) -> (continuation of Deferred@1509483278 after retry RPC after error@532381907) -> (continuation of Deferred@204817893 after retry RPC after error@1181375072) -> (continuation of Deferred@1486102832 after retry RPC after error@459968489) -> (continuation of Deferred@1029110540 after retry RPC after error@704323484) -> (continuation of Deferred@1352607282 after retry RPC after error@533065231) -> (continuation of Deferred@365029472 after retry RPC after error@1681330875) -> (continuation of Deferred@1303385785 after retry RPC after error@2109025430)))
06:34:24.169 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:24.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.190 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.190 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.191 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.228 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.228 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.268 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.268 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.268 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.328 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.329 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.329 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.329 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.330 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.368 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.368 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.370 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.370 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1544265078(state=PENDING, result=null, callback=(continuation of Deferred@1729985892 after retry RPC after error@2017372568) -> (continuation of Deferred@1776909698 after retry RPC after error@1770630628) -> (continuation of Deferred@1710427116 after retry RPC after error@1754008739) -> (continuation of Deferred@1244363101 after retry RPC after error@1860770673) -> (continuation of Deferred@322490667 after retry RPC after error@458829563) -> (continuation of Deferred@178867548 after retry RPC after error@750648811) -> (continuation of Deferred@2044317918 after retry RPC after error@732235752), errback=(continuation of Deferred@1729985892 after retry RPC after error@2017372568) -> (continuation of Deferred@1776909698 after retry RPC after error@1770630628) -> (continuation of Deferred@1710427116 after retry RPC after error@1754008739) -> (continuation of Deferred@1244363101 after retry RPC after error@1860770673) -> (continuation of Deferred@322490667 after retry RPC after error@458829563) -> (continuation of Deferred@178867548 after retry RPC after error@750648811) -> (continuation of Deferred@2044317918 after retry RPC after error@732235752)))
06:34:24.371 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=181), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1544265078(state=PENDING, result=null, callback=(continuation of Deferred@1729985892 after retry RPC after error@2017372568) -> (continuation of Deferred@1776909698 after retry RPC after error@1770630628) -> (continuation of Deferred@1710427116 after retry RPC after error@1754008739) -> (continuation of Deferred@1244363101 after retry RPC after error@1860770673) -> (continuation of Deferred@322490667 after retry RPC after error@458829563) -> (continuation of Deferred@178867548 after retry RPC after error@750648811) -> (continuation of Deferred@2044317918 after retry RPC after error@732235752), errback=(continuation of Deferred@1729985892 after retry RPC after error@2017372568) -> (continuation of Deferred@1776909698 after retry RPC after error@1770630628) -> (continuation of Deferred@1710427116 after retry RPC after error@1754008739) -> (continuation of Deferred@1244363101 after retry RPC after error@1860770673) -> (continuation of Deferred@322490667 after retry RPC after error@458829563) -> (continuation of Deferred@178867548 after retry RPC after error@750648811) -> (continuation of Deferred@2044317918 after retry RPC after error@732235752)))
06:34:24.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.451 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.469 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.488 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.488 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.488 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.488 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.610 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.610 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.610 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.610 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.611 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.611 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [99ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@169457628(state=PENDING, result=null, callback=(continuation of Deferred@1418952544 after retry RPC after error@2096304392) -> (continuation of Deferred@282574291 after retry RPC after error@1463242562) -> (continuation of Deferred@412111547 after retry RPC after error@2089774064) -> (continuation of Deferred@1360750743 after retry RPC after error@1018917033) -> (continuation of Deferred@648341519 after retry RPC after error@569624512) -> (continuation of Deferred@723930297 after retry RPC after error@1283415884) -> (continuation of Deferred@1293562460 after retry RPC after error@1919431160), errback=(continuation of Deferred@1418952544 after retry RPC after error@2096304392) -> (continuation of Deferred@282574291 after retry RPC after error@1463242562) -> (continuation of Deferred@412111547 after retry RPC after error@2089774064) -> (continuation of Deferred@1360750743 after retry RPC after error@1018917033) -> (continuation of Deferred@648341519 after retry RPC after error@569624512) -> (continuation of Deferred@723930297 after retry RPC after error@1283415884) -> (continuation of Deferred@1293562460 after retry RPC after error@1919431160)))
06:34:24.612 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [19ms] refreshing cache from master, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [19ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [21ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [39ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [79ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [99ms] refreshing cache from master, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [99ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [99ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [101ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [120ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [220ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@169457628(state=PENDING, result=null, callback=(continuation of Deferred@1418952544 after retry RPC after error@2096304392) -> (continuation of Deferred@282574291 after retry RPC after error@1463242562) -> (continuation of Deferred@412111547 after retry RPC after error@2089774064) -> (continuation of Deferred@1360750743 after retry RPC after error@1018917033) -> (continuation of Deferred@648341519 after retry RPC after error@569624512) -> (continuation of Deferred@723930297 after retry RPC after error@1283415884) -> (continuation of Deferred@1293562460 after retry RPC after error@1919431160), errback=(continuation of Deferred@1418952544 after retry RPC after error@2096304392) -> (continuation of Deferred@282574291 after retry RPC after error@1463242562) -> (continuation of Deferred@412111547 after retry RPC after error@2089774064) -> (continuation of Deferred@1360750743 after retry RPC after error@1018917033) -> (continuation of Deferred@648341519 after retry RPC after error@569624512) -> (continuation of Deferred@723930297 after retry RPC after error@1283415884) -> (continuation of Deferred@1293562460 after retry RPC after error@1919431160)))
06:34:24.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.629 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.630 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.669 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.670 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.670 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.671 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.708 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.708 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.851 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1122078853(state=PENDING, result=null, callback=(continuation of Deferred@992831983 after retry RPC after error@1418828807) -> (continuation of Deferred@491144691 after retry RPC after error@1408076646) -> (continuation of Deferred@56619749 after retry RPC after error@1534474029) -> (continuation of Deferred@1957730413 after retry RPC after error@1262066470) -> (continuation of Deferred@932020350 after retry RPC after error@1562908413) -> (continuation of Deferred@766222021 after retry RPC after error@718061821) -> (continuation of Deferred@1897479082 after retry RPC after error@1822119050), errback=(continuation of Deferred@992831983 after retry RPC after error@1418828807) -> (continuation of Deferred@491144691 after retry RPC after error@1408076646) -> (continuation of Deferred@56619749 after retry RPC after error@1534474029) -> (continuation of Deferred@1957730413 after retry RPC after error@1262066470) -> (continuation of Deferred@932020350 after retry RPC after error@1562908413) -> (continuation of Deferred@766222021 after retry RPC after error@718061821) -> (continuation of Deferred@1897479082 after retry RPC after error@1822119050)))
06:34:24.851 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [80ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [101ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1122078853(state=PENDING, result=null, callback=(continuation of Deferred@992831983 after retry RPC after error@1418828807) -> (continuation of Deferred@491144691 after retry RPC after error@1408076646) -> (continuation of Deferred@56619749 after retry RPC after error@1534474029) -> (continuation of Deferred@1957730413 after retry RPC after error@1262066470) -> (continuation of Deferred@932020350 after retry RPC after error@1562908413) -> (continuation of Deferred@766222021 after retry RPC after error@718061821) -> (continuation of Deferred@1897479082 after retry RPC after error@1822119050), errback=(continuation of Deferred@992831983 after retry RPC after error@1418828807) -> (continuation of Deferred@491144691 after retry RPC after error@1408076646) -> (continuation of Deferred@56619749 after retry RPC after error@1534474029) -> (continuation of Deferred@1957730413 after retry RPC after error@1262066470) -> (continuation of Deferred@932020350 after retry RPC after error@1562908413) -> (continuation of Deferred@766222021 after retry RPC after error@718061821) -> (continuation of Deferred@1897479082 after retry RPC after error@1822119050)))
06:34:24.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.868 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.870 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.889 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:24.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:24.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:24.989 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:24.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:24.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.030 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.030 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.031 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.129 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [203ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1417297160(state=PENDING, result=null, callback=(continuation of Deferred@644391649 after retry RPC after error@308785800) -> (continuation of Deferred@483766746 after retry RPC after error@423775541) -> (continuation of Deferred@2053016406 after retry RPC after error@1426232679) -> (continuation of Deferred@1684579708 after retry RPC after error@1875404577) -> (continuation of Deferred@1640121781 after retry RPC after error@539647303) -> (continuation of Deferred@21516155 after retry RPC after error@756400412) -> (continuation of Deferred@1577699046 after retry RPC after error@742094853) -> (continuation of Deferred@1875630446 after retry RPC after error@1817573546), errback=(continuation of Deferred@644391649 after retry RPC after error@308785800) -> (continuation of Deferred@483766746 after retry RPC after error@423775541) -> (continuation of Deferred@2053016406 after retry RPC after error@1426232679) -> (continuation of Deferred@1684579708 after retry RPC after error@1875404577) -> (continuation of Deferred@1640121781 after retry RPC after error@539647303) -> (continuation of Deferred@21516155 after retry RPC after error@756400412) -> (continuation of Deferred@1577699046 after retry RPC after error@742094853) -> (continuation of Deferred@1875630446 after retry RPC after error@1817573546)))
06:34:25.129 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:25.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.169 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.170 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.208 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.208 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.264 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:25.263799 24938 master_proxy_rpc.cc:177] Failed to determine new leader Master: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:25.265 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:25.265213 24938 meta_cache.cc:788] Timed out: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } failed: LookupRpcById timed out after deadline expired: LookupRpcById { tablet: '11b35c262ebf4dd2b9e78d6abc3dea00', attempt: 2 } passed its deadline: Timed out: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 97) passed its deadline: Not found: no leader found: ConnectToClusterRpc(addrs: 127.23.161.254:38015,127.23.161.253:45177,127.23.161.252:42373, num_attempts: 1)
06:34:25.266 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:25.266503 24938 master_proxy_rpc.cc:203] Re-attempting LookupRpcById request to leader Master (127.23.161.252:42373)
06:34:25.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.268 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.269 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.269 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.269 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.269 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.269 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.269 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.270 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.349 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.408 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@49785588(state=PENDING, result=null, callback=(continuation of Deferred@114155109 after retry RPC after error@312934946) -> (continuation of Deferred@595967769 after retry RPC after error@471470487) -> (continuation of Deferred@1951193735 after retry RPC after error@1582329918) -> (continuation of Deferred@1344496026 after retry RPC after error@477709472) -> (continuation of Deferred@250821962 after retry RPC after error@1400859946) -> (continuation of Deferred@412875118 after retry RPC after error@1309799065) -> (continuation of Deferred@1503629180 after retry RPC after error@1898037080), errback=(continuation of Deferred@114155109 after retry RPC after error@312934946) -> (continuation of Deferred@595967769 after retry RPC after error@471470487) -> (continuation of Deferred@1951193735 after retry RPC after error@1582329918) -> (continuation of Deferred@1344496026 after retry RPC after error@477709472) -> (continuation of Deferred@250821962 after retry RPC after error@1400859946) -> (continuation of Deferred@412875118 after retry RPC after error@1309799065) -> (continuation of Deferred@1503629180 after retry RPC after error@1898037080)))
06:34:25.409 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:25.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.429 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.430 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.430 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.469 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.469 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.469 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.470 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.470 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.470 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.470 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.472 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.489 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.549 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.550 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.550 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.551 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.670 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1722766235(state=PENDING, result=null, callback=(continuation of Deferred@1507336872 after retry RPC after error@984766171) -> (continuation of Deferred@1177405420 after retry RPC after error@1510111033) -> (continuation of Deferred@1121440691 after retry RPC after error@1794855356) -> (continuation of Deferred@1120004520 after retry RPC after error@187950454) -> (continuation of Deferred@2005263848 after retry RPC after error@1195588629) -> (continuation of Deferred@176296444 after retry RPC after error@504257935) -> (continuation of Deferred@1182548227 after retry RPC after error@411139661) -> (continuation of Deferred@43417936 after retry RPC after error@1855473320), errback=(continuation of Deferred@1507336872 after retry RPC after error@984766171) -> (continuation of Deferred@1177405420 after retry RPC after error@1510111033) -> (continuation of Deferred@1121440691 after retry RPC after error@1794855356) -> (continuation of Deferred@1120004520 after retry RPC after error@187950454) -> (continuation of Deferred@2005263848 after retry RPC after error@1195588629) -> (continuation of Deferred@176296444 after retry RPC after error@504257935) -> (continuation of Deferred@1182548227 after retry RPC after error@411139661) -> (continuation of Deferred@43417936 after retry RPC after error@1855473320)))
06:34:25.671 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=241), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] refreshing cache from master, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [43ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [44ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1722766235(state=PENDING, result=null, callback=(continuation of Deferred@1507336872 after retry RPC after error@984766171) -> (continuation of Deferred@1177405420 after retry RPC after error@1510111033) -> (continuation of Deferred@1121440691 after retry RPC after error@1794855356) -> (continuation of Deferred@1120004520 after retry RPC after error@187950454) -> (continuation of Deferred@2005263848 after retry RPC after error@1195588629) -> (continuation of Deferred@176296444 after retry RPC after error@504257935) -> (continuation of Deferred@1182548227 after retry RPC after error@411139661) -> (continuation of Deferred@43417936 after retry RPC after error@1855473320), errback=(continuation of Deferred@1507336872 after retry RPC after error@984766171) -> (continuation of Deferred@1177405420 after retry RPC after error@1510111033) -> (continuation of Deferred@1121440691 after retry RPC after error@1794855356) -> (continuation of Deferred@1120004520 after retry RPC after error@187950454) -> (continuation of Deferred@2005263848 after retry RPC after error@1195588629) -> (continuation of Deferred@176296444 after retry RPC after error@504257935) -> (continuation of Deferred@1182548227 after retry RPC after error@411139661) -> (continuation of Deferred@43417936 after retry RPC after error@1855473320)))
06:34:25.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.689 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.729 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.749 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.749 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.770 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.809 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.810 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.810 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.810 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.889 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.889 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.890 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.890 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@66789521(state=PENDING, result=null, callback=(continuation of Deferred@1175772715 after retry RPC after error@951780902) -> (continuation of Deferred@1779971148 after retry RPC after error@809336374) -> (continuation of Deferred@1345854364 after retry RPC after error@810728843) -> (continuation of Deferred@465786265 after retry RPC after error@1087656985) -> (continuation of Deferred@1344295780 after retry RPC after error@1966805518) -> (continuation of Deferred@1951788717 after retry RPC after error@373491874) -> (continuation of Deferred@1143652989 after retry RPC after error@1764496276), errback=(continuation of Deferred@1175772715 after retry RPC after error@951780902) -> (continuation of Deferred@1779971148 after retry RPC after error@809336374) -> (continuation of Deferred@1345854364 after retry RPC after error@810728843) -> (continuation of Deferred@465786265 after retry RPC after error@1087656985) -> (continuation of Deferred@1344295780 after retry RPC after error@1966805518) -> (continuation of Deferred@1951788717 after retry RPC after error@373491874) -> (continuation of Deferred@1143652989 after retry RPC after error@1764496276)))
06:34:25.891 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=201), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@66789521(state=PENDING, result=null, callback=(continuation of Deferred@1175772715 after retry RPC after error@951780902) -> (continuation of Deferred@1779971148 after retry RPC after error@809336374) -> (continuation of Deferred@1345854364 after retry RPC after error@810728843) -> (continuation of Deferred@465786265 after retry RPC after error@1087656985) -> (continuation of Deferred@1344295780 after retry RPC after error@1966805518) -> (continuation of Deferred@1951788717 after retry RPC after error@373491874) -> (continuation of Deferred@1143652989 after retry RPC after error@1764496276), errback=(continuation of Deferred@1175772715 after retry RPC after error@951780902) -> (continuation of Deferred@1779971148 after retry RPC after error@809336374) -> (continuation of Deferred@1345854364 after retry RPC after error@810728843) -> (continuation of Deferred@465786265 after retry RPC after error@1087656985) -> (continuation of Deferred@1344295780 after retry RPC after error@1966805518) -> (continuation of Deferred@1951788717 after retry RPC after error@373491874) -> (continuation of Deferred@1143652989 after retry RPC after error@1764496276)))
06:34:25.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.908 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.908 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.908 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.929 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.948 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.950 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.969 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:25.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:25.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:25.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:25.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:25.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.029 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.030 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.030 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.070 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.070 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.070 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.151 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1042913699(state=PENDING, result=null, callback=(continuation of Deferred@1182457399 after retry RPC after error@1425892066) -> (continuation of Deferred@9719726 after retry RPC after error@1912744606) -> (continuation of Deferred@1645857167 after retry RPC after error@1573670236) -> (continuation of Deferred@2139586676 after retry RPC after error@481428416) -> (continuation of Deferred@818668648 after retry RPC after error@265480576) -> (continuation of Deferred@1276468270 after retry RPC after error@1077275247) -> (continuation of Deferred@800239269 after retry RPC after error@926211130), errback=(continuation of Deferred@1182457399 after retry RPC after error@1425892066) -> (continuation of Deferred@9719726 after retry RPC after error@1912744606) -> (continuation of Deferred@1645857167 after retry RPC after error@1573670236) -> (continuation of Deferred@2139586676 after retry RPC after error@481428416) -> (continuation of Deferred@818668648 after retry RPC after error@265480576) -> (continuation of Deferred@1276468270 after retry RPC after error@1077275247) -> (continuation of Deferred@800239269 after retry RPC after error@926211130)))
06:34:26.151 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=242), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [240ms] refreshing cache from master, [240ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [241ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [241ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [242ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [242ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1042913699(state=PENDING, result=null, callback=(continuation of Deferred@1182457399 after retry RPC after error@1425892066) -> (continuation of Deferred@9719726 after retry RPC after error@1912744606) -> (continuation of Deferred@1645857167 after retry RPC after error@1573670236) -> (continuation of Deferred@2139586676 after retry RPC after error@481428416) -> (continuation of Deferred@818668648 after retry RPC after error@265480576) -> (continuation of Deferred@1276468270 after retry RPC after error@1077275247) -> (continuation of Deferred@800239269 after retry RPC after error@926211130), errback=(continuation of Deferred@1182457399 after retry RPC after error@1425892066) -> (continuation of Deferred@9719726 after retry RPC after error@1912744606) -> (continuation of Deferred@1645857167 after retry RPC after error@1573670236) -> (continuation of Deferred@2139586676 after retry RPC after error@481428416) -> (continuation of Deferred@818668648 after retry RPC after error@265480576) -> (continuation of Deferred@1276468270 after retry RPC after error@1077275247) -> (continuation of Deferred@800239269 after retry RPC after error@926211130)))
06:34:26.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.169 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.208 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.209 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.289 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.289 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.289 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.389 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.390 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.390 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.391 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.391 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1491828495(state=PENDING, result=null, callback=(continuation of Deferred@627890241 after retry RPC after error@937757482) -> (continuation of Deferred@121169963 after retry RPC after error@69790916) -> (continuation of Deferred@1204126416 after retry RPC after error@1491500377) -> (continuation of Deferred@417316830 after retry RPC after error@1532116685) -> (continuation of Deferred@812315765 after retry RPC after error@1527597972) -> (continuation of Deferred@1053372004 after retry RPC after error@1948764966) -> (continuation of Deferred@327762059 after retry RPC after error@585754757), errback=(continuation of Deferred@627890241 after retry RPC after error@937757482) -> (continuation of Deferred@121169963 after retry RPC after error@69790916) -> (continuation of Deferred@1204126416 after retry RPC after error@1491500377) -> (continuation of Deferred@417316830 after retry RPC after error@1532116685) -> (continuation of Deferred@812315765 after retry RPC after error@1527597972) -> (continuation of Deferred@1053372004 after retry RPC after error@1948764966) -> (continuation of Deferred@327762059 after retry RPC after error@585754757)))
06:34:26.392 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [1ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [180ms] refreshing cache from master, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [180ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [181ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [182ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [182ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1491828495(state=PENDING, result=null, callback=(continuation of Deferred@627890241 after retry RPC after error@937757482) -> (continuation of Deferred@121169963 after retry RPC after error@69790916) -> (continuation of Deferred@1204126416 after retry RPC after error@1491500377) -> (continuation of Deferred@417316830 after retry RPC after error@1532116685) -> (continuation of Deferred@812315765 after retry RPC after error@1527597972) -> (continuation of Deferred@1053372004 after retry RPC after error@1948764966) -> (continuation of Deferred@327762059 after retry RPC after error@585754757), errback=(continuation of Deferred@627890241 after retry RPC after error@937757482) -> (continuation of Deferred@121169963 after retry RPC after error@69790916) -> (continuation of Deferred@1204126416 after retry RPC after error@1491500377) -> (continuation of Deferred@417316830 after retry RPC after error@1532116685) -> (continuation of Deferred@812315765 after retry RPC after error@1527597972) -> (continuation of Deferred@1053372004 after retry RPC after error@1948764966) -> (continuation of Deferred@327762059 after retry RPC after error@585754757)))
06:34:26.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.428 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.428 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.468 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.468 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.468 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.468 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.469 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.469 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.469 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.470 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.491 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=36, TimeoutTracker(timeout=49956, elapsed=47141), Traces: [0ms] sending RPC to server master-127.23.161.252:42373, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] refreshing cache from master, [20ms] sending RPC to server master-127.23.161.252:42373, [20ms] sending RPC to server master-127.23.161.254:38015, [21ms] sending RPC to server master-127.23.161.253:45177, [21ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] sending RPC to server master-127.23.161.252:42373, [39ms] sending RPC to server master-127.23.161.254:38015, [40ms] sending RPC to server master-127.23.161.253:45177, [40ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] sending RPC to server master-127.23.161.252:42373, [60ms] sending RPC to server master-127.23.161.254:38015, [60ms] sending RPC to server master-127.23.161.253:45177, [60ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] sending RPC to server master-127.23.161.252:42373, [79ms] sending RPC to server master-127.23.161.254:38015, [80ms] sending RPC to server master-127.23.161.253:45177, [80ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] sending RPC to server master-127.23.161.252:42373, [119ms] sending RPC to server master-127.23.161.254:38015, [119ms] sending RPC to server master-127.23.161.253:45177, [120ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] sending RPC to server master-127.23.161.252:42373, [141ms] sending RPC to server master-127.23.161.254:38015, [141ms] sending RPC to server master-127.23.161.253:45177, [141ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] sending RPC to server master-127.23.161.252:42373, [219ms] sending RPC to server master-127.23.161.254:38015, [220ms] sending RPC to server master-127.23.161.253:45177, [220ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [299ms] refreshing cache from master, [300ms] sending RPC to server master-127.23.161.252:42373, [300ms] sending RPC to server master-127.23.161.254:38015, [300ms] sending RPC to server master-127.23.161.253:45177, [301ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [301ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [303ms] received response from server master-127.23.161.253:45177: OK, [303ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [499ms] refreshing cache from master, [499ms] sending RPC to server master-127.23.161.252:42373, [500ms] sending RPC to server master-127.23.161.254:38015, [500ms] sending RPC to server master-127.23.161.253:45177, [501ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [501ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [502ms] received response from server master-127.23.161.253:45177: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1339ms] refreshing cache from master, [1339ms] sending RPC to server master-127.23.161.252:42373, [1340ms] sending RPC to server master-127.23.161.254:38015, [1340ms] sending RPC to server master-127.23.161.253:45177, [1340ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1341ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1341ms] received response from server master-127.23.161.253:45177: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1479ms] refreshing cache from master, [1479ms] sending RPC to server master-127.23.161.252:42373, [1479ms] sending RPC to server master-127.23.161.254:38015, [1479ms] sending RPC to server master-127.23.161.253:45177, [1480ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1480ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1481ms] received response from server master-127.23.161.253:45177: OK, [1481ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4679ms] refreshing cache from master, [4679ms] sending RPC to server master-127.23.161.252:42373, [4680ms] sending RPC to server master-127.23.161.254:38015, [4680ms] sending RPC to server master-127.23.161.253:45177, [4680ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [4681ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4681ms] received response from server master-127.23.161.253:45177: OK, [4681ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4919ms] refreshing cache from master, [4919ms] sending RPC to server master-127.23.161.252:42373, [4920ms] too many traces: truncated at 100 traces, deferred=Deferred@1771827691(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.252:42373 -> (continuation of Deferred@2027298585 after retry RPC after error@1476666133) -> (continuation of Deferred@1186046597 after retry RPC after error@1880299281) -> (continuation of Deferred@955642077 after retry RPC after error@1091592285) -> (continuation of Deferred@176052080 after retry RPC after error@1656578737) -> (continuation of Deferred@1968768291 after retry RPC after error@923369821) -> (continuation of Deferred@1328842011 after retry RPC after error@1175834278) -> (continuation of Deferred@2069412698 after retry RPC after error@10538501) -> (continuation of Deferred@1800900439 after retry RPC after error@56560741) -> (continuation of Deferred@1038349631 after retry RPC after error@993383520) -> (continuation of Deferred@1817172326 after retry RPC after error@386456750) -> (continuation of Deferred@903907709 after retry RPC after error@543411573) -> (continuation of Deferred@846570742 after retry RPC after error@1665790906) -> (continuation of Deferred@1389573583 after retry RPC after error@1826968197) -> (continuation of Deferred@1940396539 after retry RPC after error@1599135767) -> (continuation of Deferred@350673791 after retry RPC after error@1389478428) -> (continuation of Deferred@1636138620 after retry RPC after error@957180419) -> (continuation of Deferred@462582654 after retry RPC after error@1192198962) -> (continuation of Deferred@229864241 after retry RPC after error@1606180342) -> (continuation of Deferred@1472060668 after retry RPC after error@1813041962) -> (continuation of Deferred@2058100578 after retry RPC after error@1195552300) -> (continuation of Deferred@2124533764 after retry RPC after error@339924345) -> (continuation of Deferred@235900828 after retry RPC after error@1559131071) -> (continuation of Deferred@750762454 after retry RPC after error@777279525) -> (continuation of Deferred@1549095196 after retry RPC after error@1938404002) -> (continuation of Deferred@581551116 after retry RPC after error@774907148) -> (continuation of Deferred@1486572647 after retry RPC after error@1723464052) -> (continuation of Deferred@173856779 after retry RPC after error@636191753) -> (continuation of Deferred@663821479 after retry RPC after error@239743864) -> (continuation of Deferred@1647984186 after retry RPC after error@453522872) -> (continuation of Deferred@886273626 after retry RPC after error@1809666635) -> (continuation of Deferred@1487235812 after retry RPC after error@47854013) -> (continuation of Deferred@756344346 after retry RPC after error@744138489) -> (continuation of Deferred@1608195036 after retry RPC after error@215981635) -> (continuation of Deferred@108105204 after retry RPC after error@2145754797), errback=org.apache.kudu.client.ConnectToCluster$1@78676f76 -> ConnectToMasterErrCB for 127.23.161.252:42373 -> (continuation of Deferred@2027298585 after retry RPC after error@1476666133) -> (continuation of Deferred@1186046597 after retry RPC after error@1880299281) -> (continuation of Deferred@955642077 after retry RPC after error@1091592285) -> (continuation of Deferred@176052080 after retry RPC after error@1656578737) -> (continuation of Deferred@1968768291 after retry RPC after error@923369821) -> (continuation of Deferred@1328842011 after retry RPC after error@1175834278) -> (continuation of Deferred@2069412698 after retry RPC after error@10538501) -> (continuation of Deferred@1800900439 after retry RPC after error@56560741) -> (continuation of Deferred@1038349631 after retry RPC after error@993383520) -> (continuation of Deferred@1817172326 after retry RPC after error@386456750) -> (continuation of Deferred@903907709 after retry RPC after error@543411573) -> (continuation of Deferred@846570742 after retry RPC after error@1665790906) -> (continuation of Deferred@1389573583 after retry RPC after error@1826968197) -> (continuation of Deferred@1940396539 after retry RPC after error@1599135767) -> (continuation of Deferred@350673791 after retry RPC after error@1389478428) -> (continuation of Deferred@1636138620 after retry RPC after error@957180419) -> (continuation of Deferred@462582654 after retry RPC after error@1192198962) -> (continuation of Deferred@229864241 after retry RPC after error@1606180342) -> (continuation of Deferred@1472060668 after retry RPC after error@1813041962) -> (continuation of Deferred@2058100578 after retry RPC after error@1195552300) -> (continuation of Deferred@2124533764 after retry RPC after error@339924345) -> (continuation of Deferred@235900828 after retry RPC after error@1559131071) -> (continuation of Deferred@750762454 after retry RPC after error@777279525) -> (continuation of Deferred@1549095196 after retry RPC after error@1938404002) -> (continuation of Deferred@581551116 after retry RPC after error@774907148) -> (continuation of Deferred@1486572647 after retry RPC after error@1723464052) -> (continuation of Deferred@173856779 after retry RPC after error@636191753) -> (continuation of Deferred@663821479 after retry RPC after error@239743864) -> (continuation of Deferred@1647984186 after retry RPC after error@453522872) -> (continuation of Deferred@886273626 after retry RPC after error@1809666635) -> (continuation of Deferred@1487235812 after retry RPC after error@47854013) -> (continuation of Deferred@756344346 after retry RPC after error@744138489) -> (continuation of Deferred@1608195036 after retry RPC after error@215981635) -> (continuation of Deferred@108105204 after retry RPC after error@2145754797)))
06:34:26.491 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=36, TimeoutTracker(timeout=49956, elapsed=47141), Traces: [0ms] sending RPC to server master-127.23.161.252:42373, [0ms] delaying RPC due to: Illegal state: connection is terminated, [19ms] refreshing cache from master, [20ms] sending RPC to server master-127.23.161.252:42373, [20ms] sending RPC to server master-127.23.161.254:38015, [21ms] sending RPC to server master-127.23.161.253:45177, [21ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [39ms] refreshing cache from master, [39ms] sending RPC to server master-127.23.161.252:42373, [39ms] sending RPC to server master-127.23.161.254:38015, [40ms] sending RPC to server master-127.23.161.253:45177, [40ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [40ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [59ms] refreshing cache from master, [59ms] sending RPC to server master-127.23.161.252:42373, [60ms] sending RPC to server master-127.23.161.254:38015, [60ms] sending RPC to server master-127.23.161.253:45177, [60ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] received response from server master-127.23.161.253:45177: OK, [61ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [79ms] refreshing cache from master, [79ms] sending RPC to server master-127.23.161.252:42373, [79ms] sending RPC to server master-127.23.161.254:38015, [80ms] sending RPC to server master-127.23.161.253:45177, [80ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [119ms] refreshing cache from master, [119ms] sending RPC to server master-127.23.161.252:42373, [119ms] sending RPC to server master-127.23.161.254:38015, [119ms] sending RPC to server master-127.23.161.253:45177, [120ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] received response from server master-127.23.161.253:45177: OK, [121ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [141ms] refreshing cache from master, [141ms] sending RPC to server master-127.23.161.252:42373, [141ms] sending RPC to server master-127.23.161.254:38015, [141ms] sending RPC to server master-127.23.161.253:45177, [141ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [143ms] received response from server master-127.23.161.253:45177: OK, [143ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [219ms] refreshing cache from master, [219ms] sending RPC to server master-127.23.161.252:42373, [219ms] sending RPC to server master-127.23.161.254:38015, [220ms] sending RPC to server master-127.23.161.253:45177, [220ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [299ms] refreshing cache from master, [300ms] sending RPC to server master-127.23.161.252:42373, [300ms] sending RPC to server master-127.23.161.254:38015, [300ms] sending RPC to server master-127.23.161.253:45177, [301ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [301ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [303ms] received response from server master-127.23.161.253:45177: OK, [303ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [499ms] refreshing cache from master, [499ms] sending RPC to server master-127.23.161.252:42373, [500ms] sending RPC to server master-127.23.161.254:38015, [500ms] sending RPC to server master-127.23.161.253:45177, [501ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [501ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [502ms] received response from server master-127.23.161.253:45177: OK, [502ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1339ms] refreshing cache from master, [1339ms] sending RPC to server master-127.23.161.252:42373, [1340ms] sending RPC to server master-127.23.161.254:38015, [1340ms] sending RPC to server master-127.23.161.253:45177, [1340ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1341ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1341ms] received response from server master-127.23.161.253:45177: OK, [1342ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1479ms] refreshing cache from master, [1479ms] sending RPC to server master-127.23.161.252:42373, [1479ms] sending RPC to server master-127.23.161.254:38015, [1479ms] sending RPC to server master-127.23.161.253:45177, [1480ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1480ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1481ms] received response from server master-127.23.161.253:45177: OK, [1481ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4679ms] refreshing cache from master, [4679ms] sending RPC to server master-127.23.161.252:42373, [4680ms] sending RPC to server master-127.23.161.254:38015, [4680ms] sending RPC to server master-127.23.161.253:45177, [4680ms] received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [4681ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4681ms] received response from server master-127.23.161.253:45177: OK, [4681ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4919ms] refreshing cache from master, [4919ms] sending RPC to server master-127.23.161.252:42373, [4920ms] too many traces: truncated at 100 traces, deferred=Deferred@1771827691(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.252:42373 -> (continuation of Deferred@2027298585 after retry RPC after error@1476666133) -> (continuation of Deferred@1186046597 after retry RPC after error@1880299281) -> (continuation of Deferred@955642077 after retry RPC after error@1091592285) -> (continuation of Deferred@176052080 after retry RPC after error@1656578737) -> (continuation of Deferred@1968768291 after retry RPC after error@923369821) -> (continuation of Deferred@1328842011 after retry RPC after error@1175834278) -> (continuation of Deferred@2069412698 after retry RPC after error@10538501) -> (continuation of Deferred@1800900439 after retry RPC after error@56560741) -> (continuation of Deferred@1038349631 after retry RPC after error@993383520) -> (continuation of Deferred@1817172326 after retry RPC after error@386456750) -> (continuation of Deferred@903907709 after retry RPC after error@543411573) -> (continuation of Deferred@846570742 after retry RPC after error@1665790906) -> (continuation of Deferred@1389573583 after retry RPC after error@1826968197) -> (continuation of Deferred@1940396539 after retry RPC after error@1599135767) -> (continuation of Deferred@350673791 after retry RPC after error@1389478428) -> (continuation of Deferred@1636138620 after retry RPC after error@957180419) -> (continuation of Deferred@462582654 after retry RPC after error@1192198962) -> (continuation of Deferred@229864241 after retry RPC after error@1606180342) -> (continuation of Deferred@1472060668 after retry RPC after error@1813041962) -> (continuation of Deferred@2058100578 after retry RPC after error@1195552300) -> (continuation of Deferred@2124533764 after retry RPC after error@339924345) -> (continuation of Deferred@235900828 after retry RPC after error@1559131071) -> (continuation of Deferred@750762454 after retry RPC after error@777279525) -> (continuation of Deferred@1549095196 after retry RPC after error@1938404002) -> (continuation of Deferred@581551116 after retry RPC after error@774907148) -> (continuation of Deferred@1486572647 after retry RPC after error@1723464052) -> (continuation of Deferred@173856779 after retry RPC after error@636191753) -> (continuation of Deferred@663821479 after retry RPC after error@239743864) -> (continuation of Deferred@1647984186 after retry RPC after error@453522872) -> (continuation of Deferred@886273626 after retry RPC after error@1809666635) -> (continuation of Deferred@1487235812 after retry RPC after error@47854013) -> (continuation of Deferred@756344346 after retry RPC after error@744138489) -> (continuation of Deferred@1608195036 after retry RPC after error@215981635) -> (continuation of Deferred@108105204 after retry RPC after error@2145754797), errback=org.apache.kudu.client.ConnectToCluster$1@78676f76 -> ConnectToMasterErrCB for 127.23.161.252:42373 -> (continuation of Deferred@2027298585 after retry RPC after error@1476666133) -> (continuation of Deferred@1186046597 after retry RPC after error@1880299281) -> (continuation of Deferred@955642077 after retry RPC after error@1091592285) -> (continuation of Deferred@176052080 after retry RPC after error@1656578737) -> (continuation of Deferred@1968768291 after retry RPC after error@923369821) -> (continuation of Deferred@1328842011 after retry RPC after error@1175834278) -> (continuation of Deferred@2069412698 after retry RPC after error@10538501) -> (continuation of Deferred@1800900439 after retry RPC after error@56560741) -> (continuation of Deferred@1038349631 after retry RPC after error@993383520) -> (continuation of Deferred@1817172326 after retry RPC after error@386456750) -> (continuation of Deferred@903907709 after retry RPC after error@543411573) -> (continuation of Deferred@846570742 after retry RPC after error@1665790906) -> (continuation of Deferred@1389573583 after retry RPC after error@1826968197) -> (continuation of Deferred@1940396539 after retry RPC after error@1599135767) -> (continuation of Deferred@350673791 after retry RPC after error@1389478428) -> (continuation of Deferred@1636138620 after retry RPC after error@957180419) -> (continuation of Deferred@462582654 after retry RPC after error@1192198962) -> (continuation of Deferred@229864241 after retry RPC after error@1606180342) -> (continuation of Deferred@1472060668 after retry RPC after error@1813041962) -> (continuation of Deferred@2058100578 after retry RPC after error@1195552300) -> (continuation of Deferred@2124533764 after retry RPC after error@339924345) -> (continuation of Deferred@235900828 after retry RPC after error@1559131071) -> (continuation of Deferred@750762454 after retry RPC after error@777279525) -> (continuation of Deferred@1549095196 after retry RPC after error@1938404002) -> (continuation of Deferred@581551116 after retry RPC after error@774907148) -> (continuation of Deferred@1486572647 after retry RPC after error@1723464052) -> (continuation of Deferred@173856779 after retry RPC after error@636191753) -> (continuation of Deferred@663821479 after retry RPC after error@239743864) -> (continuation of Deferred@1647984186 after retry RPC after error@453522872) -> (continuation of Deferred@886273626 after retry RPC after error@1809666635) -> (continuation of Deferred@1487235812 after retry RPC after error@47854013) -> (continuation of Deferred@756344346 after retry RPC after error@744138489) -> (continuation of Deferred@1608195036 after retry RPC after error@215981635) -> (continuation of Deferred@108105204 after retry RPC after error@2145754797)))
06:34:26.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.492 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.508 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.508 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.508 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.509 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.509 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.510 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.549 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.549 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.570 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.570 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.628 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.629 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.629 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.629 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.629 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.629 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.629 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.630 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.630 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@372783686(state=PENDING, result=null, callback=(continuation of Deferred@661268637 after retry RPC after error@1268961994) -> (continuation of Deferred@1624563902 after retry RPC after error@929635560) -> (continuation of Deferred@1102177368 after retry RPC after error@483940902) -> (continuation of Deferred@1531877395 after retry RPC after error@608125390) -> (continuation of Deferred@1784865333 after retry RPC after error@884313141) -> (continuation of Deferred@454300848 after retry RPC after error@1165690669) -> (continuation of Deferred@947201789 after retry RPC after error@549044706), errback=(continuation of Deferred@661268637 after retry RPC after error@1268961994) -> (continuation of Deferred@1624563902 after retry RPC after error@929635560) -> (continuation of Deferred@1102177368 after retry RPC after error@483940902) -> (continuation of Deferred@1531877395 after retry RPC after error@608125390) -> (continuation of Deferred@1784865333 after retry RPC after error@884313141) -> (continuation of Deferred@454300848 after retry RPC after error@1165690669) -> (continuation of Deferred@947201789 after retry RPC after error@549044706)))
06:34:26.631 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [60ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] refreshing cache from master, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [84ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@372783686(state=PENDING, result=null, callback=(continuation of Deferred@661268637 after retry RPC after error@1268961994) -> (continuation of Deferred@1624563902 after retry RPC after error@929635560) -> (continuation of Deferred@1102177368 after retry RPC after error@483940902) -> (continuation of Deferred@1531877395 after retry RPC after error@608125390) -> (continuation of Deferred@1784865333 after retry RPC after error@884313141) -> (continuation of Deferred@454300848 after retry RPC after error@1165690669) -> (continuation of Deferred@947201789 after retry RPC after error@549044706), errback=(continuation of Deferred@661268637 after retry RPC after error@1268961994) -> (continuation of Deferred@1624563902 after retry RPC after error@929635560) -> (continuation of Deferred@1102177368 after retry RPC after error@483940902) -> (continuation of Deferred@1531877395 after retry RPC after error@608125390) -> (continuation of Deferred@1784865333 after retry RPC after error@884313141) -> (continuation of Deferred@454300848 after retry RPC after error@1165690669) -> (continuation of Deferred@947201789 after retry RPC after error@549044706)))
06:34:26.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.649 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.668 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.669 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.669 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.669 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.669 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.669 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.669 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.670 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.688 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.688 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.688 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.688 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.689 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.708 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.709 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.709 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.709 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.710 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.728 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.728 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.729 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.770 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.808 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.809 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.809 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.809 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.809 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.809 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.810 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.810 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.811 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.868 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.869 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.869 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.869 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.869 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.869 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.869 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.869 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.871 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.871 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@257785721(state=PENDING, result=null, callback=(continuation of Deferred@202177465 after retry RPC after error@275265448) -> (continuation of Deferred@360911922 after retry RPC after error@1900087116) -> (continuation of Deferred@452717550 after retry RPC after error@1275021934) -> (continuation of Deferred@1188099495 after retry RPC after error@1070788950) -> (continuation of Deferred@2058607065 after retry RPC after error@1261322459) -> (continuation of Deferred@169015764 after retry RPC after error@576905601) -> (continuation of Deferred@1190889733 after retry RPC after error@2146050457), errback=(continuation of Deferred@202177465 after retry RPC after error@275265448) -> (continuation of Deferred@360911922 after retry RPC after error@1900087116) -> (continuation of Deferred@452717550 after retry RPC after error@1275021934) -> (continuation of Deferred@1188099495 after retry RPC after error@1070788950) -> (continuation of Deferred@2058607065 after retry RPC after error@1261322459) -> (continuation of Deferred@169015764 after retry RPC after error@576905601) -> (continuation of Deferred@1190889733 after retry RPC after error@2146050457)))
06:34:26.871 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [40ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [81ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [161ms] refreshing cache from master, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [163ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [163ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@257785721(state=PENDING, result=null, callback=(continuation of Deferred@202177465 after retry RPC after error@275265448) -> (continuation of Deferred@360911922 after retry RPC after error@1900087116) -> (continuation of Deferred@452717550 after retry RPC after error@1275021934) -> (continuation of Deferred@1188099495 after retry RPC after error@1070788950) -> (continuation of Deferred@2058607065 after retry RPC after error@1261322459) -> (continuation of Deferred@169015764 after retry RPC after error@576905601) -> (continuation of Deferred@1190889733 after retry RPC after error@2146050457), errback=(continuation of Deferred@202177465 after retry RPC after error@275265448) -> (continuation of Deferred@360911922 after retry RPC after error@1900087116) -> (continuation of Deferred@452717550 after retry RPC after error@1275021934) -> (continuation of Deferred@1188099495 after retry RPC after error@1070788950) -> (continuation of Deferred@2058607065 after retry RPC after error@1261322459) -> (continuation of Deferred@169015764 after retry RPC after error@576905601) -> (continuation of Deferred@1190889733 after retry RPC after error@2146050457)))
06:34:26.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.888 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.888 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.888 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.889 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.889 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.889 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.889 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.889 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.909 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.929 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.950 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.950 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.950 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.950 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:26.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:26.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:26.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:26.989 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:26.990 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.009 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.029 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.029 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.029 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.029 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.108 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.109 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.109 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.109 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.109 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.111 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.112 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.113 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.113 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.149 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [0ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] refreshing cache from master, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [63ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [224ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [225ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [225ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1057225618(state=PENDING, result=null, callback=(continuation of Deferred@220292911 after retry RPC after error@1326997182) -> (continuation of Deferred@17126572 after retry RPC after error@561700981) -> (continuation of Deferred@660927338 after retry RPC after error@713335958) -> (continuation of Deferred@17769822 after retry RPC after error@1996489794) -> (continuation of Deferred@2004829803 after retry RPC after error@2018573318) -> (continuation of Deferred@9245575 after retry RPC after error@2002265536) -> (continuation of Deferred@2056874334 after retry RPC after error@89529031) -> (continuation of Deferred@1831805121 after retry RPC after error@1534033537), errback=(continuation of Deferred@220292911 after retry RPC after error@1326997182) -> (continuation of Deferred@17126572 after retry RPC after error@561700981) -> (continuation of Deferred@660927338 after retry RPC after error@713335958) -> (continuation of Deferred@17769822 after retry RPC after error@1996489794) -> (continuation of Deferred@2004829803 after retry RPC after error@2018573318) -> (continuation of Deferred@9245575 after retry RPC after error@2002265536) -> (continuation of Deferred@2056874334 after retry RPC after error@89529031) -> (continuation of Deferred@1831805121 after retry RPC after error@1534033537)))
06:34:27.149 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:27.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.169 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.169 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.169 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.169 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.169 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.171 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.228 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.228 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.228 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.229 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.229 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.229 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.230 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.249 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.249 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.250 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.290 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.290 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.290 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:27.290 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.329 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:27.329380 25314 master_runner.cc:186] Error getting master registration for 127.23.161.254:38015: Network error: Client connection negotiation failed: client connection to 127.23.161.254:38015: connect: Connection refused (error 111),
06:34:27.329 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.329 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:27.329624 25314 master_runner.cc:418] Couldn't verify the masters in the cluster. Trying again...
06:34:27.329 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.329 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.408 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.428 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [161ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=Deferred@1208627573(state=PENDING, result=null, callback=(continuation of Deferred@41280596 after retry RPC after error@1192687981) -> (continuation of Deferred@738991211 after retry RPC after error@898896134) -> (continuation of Deferred@418256036 after retry RPC after error@1749240645) -> (continuation of Deferred@610336098 after retry RPC after error@829015241) -> (continuation of Deferred@1506458996 after retry RPC after error@723639403) -> (continuation of Deferred@143245184 after retry RPC after error@2008645076), errback=(continuation of Deferred@41280596 after retry RPC after error@1192687981) -> (continuation of Deferred@738991211 after retry RPC after error@898896134) -> (continuation of Deferred@418256036 after retry RPC after error@1749240645) -> (continuation of Deferred@610336098 after retry RPC after error@829015241) -> (continuation of Deferred@1506458996 after retry RPC after error@723639403) -> (continuation of Deferred@143245184 after retry RPC after error@2008645076)))
06:34:27.429 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=90, elapsed=100), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@2064928586(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@5ef316c7 -> ConnectToMasterErrCB for 127.23.161.254:38015))
06:34:27.429 [WARN - pool-49-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.448 [WARN - pool-49-thread-1] (HashedWheelTimer.java:718) An exception was thrown by TimerTask.
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198) ~[main/:?]
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464) ~[main/:?]
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343) ~[main/:?]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503) [netty-common-4.1.94.Final.jar:4.1.94.Final]
at java.lang.Thread.run(Thread.java:750) [?:1.8.0_362]
java.lang.IllegalStateException: This tracker doesn't have a timeout set so it cannot answer getMillisBeforeTimeout()
at org.apache.kudu.client.TimeoutTracker.getMillisBeforeTimeout(TimeoutTracker.java:86)
at org.apache.kudu.client.ConnectToCluster.connectToMaster(ConnectToCluster.java:106)
at org.apache.kudu.client.ConnectToCluster.connectToMasters(ConnectToCluster.java:198)
at org.apache.kudu.client.ConnectToCluster.run(ConnectToCluster.java:158)
at org.apache.kudu.client.AsyncKuduClient.getMasterTableLocationsPB(AsyncKuduClient.java:1953)
at org.apache.kudu.client.AsyncKuduClient.locateTablet(AsyncKuduClient.java:1923)
at org.apache.kudu.client.AsyncKuduClient.sendRpcToTablet(AsyncKuduClient.java:1464)
at org.apache.kudu.client.AsyncKuduClient.lambda$delayedSendRpcToTablet$18(AsyncKuduClient.java:2343)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.run(HashedWheelTimer.java:715)
at io.netty.util.concurrent.ImmediateExecutor.execute(ImmediateExecutor.java:34)
at io.netty.util.HashedWheelTimer$HashedWheelTimeout.expire(HashedWheelTimer.java:703)
at io.netty.util.HashedWheelTimer$HashedWheelBucket.expireTimeouts(HashedWheelTimer.java:790)
at io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:503)
at java.lang.Thread.run(Thread.java:750)
06:34:27.448 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.708 [DEBUG - pool-49-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=null) with no deferred attached!
06:34:27.709 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@2115972071(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@381c4fcd -> ConnectToMasterErrCB for 127.23.161.254:38015))
06:34:27.709 [WARN - pool-49-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.709 [DEBUG - pool-49-thread-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=261), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@2115972071(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@381c4fcd -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:27.709 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=261), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@2115972071(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@381c4fcd -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:27.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:27.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:27.988 [DEBUG - pool-49-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=null) with no deferred attached!
06:34:27.989 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@1490841090(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@5f1b26ed -> ConnectToMasterErrCB for 127.23.161.254:38015))
06:34:27.989 [WARN - pool-49-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:27.989 [DEBUG - pool-49-thread-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@1490841090(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@5f1b26ed -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:27.989 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@1490841090(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@5f1b26ed -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:28.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.268 [DEBUG - pool-49-thread-1] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, deferred=null) with no deferred attached!
06:34:28.268 [INFO - pool-49-thread-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@575639282(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@19d858c7 -> ConnectToMasterErrCB for 127.23.161.254:38015))
06:34:28.269 [WARN - pool-49-thread-1] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.269 [DEBUG - pool-49-thread-1] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@575639282(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@19d858c7 -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:28.269 [DEBUG - pool-49-thread-1] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=260), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [261ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=ConnectToMaster, tablet=null, attempt=1, TimeoutTracker(timeout=250, elapsed=259), Traces: [0ms] sending RPC to server master-127.23.161.254:38015, deferred=Deferred@575639282(state=PENDING, result=null, callback=passthrough -> ConnectToMasterCB for 127.23.161.254:38015, errback=org.apache.kudu.client.ConnectToCluster$1@19d858c7 -> ConnectToMasterErrCB for 127.23.161.254:38015)), deferred=null)
06:34:28.288 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.289 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.289 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.289 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.332 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.332 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.332 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ConnectToMaster, tablet=null, attempt=0, TimeoutTracker(timeout=0, elapsed=903), Traces: [0ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=null) with no deferred attached!
06:34:28.332 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.332 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.332 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ConnectToMaster, tablet=null, attempt=0, TimeoutTracker(timeout=0, elapsed=623), Traces: [0ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=null) with no deferred attached!
06:34:28.333 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ConnectToMaster, tablet=null, attempt=0, TimeoutTracker(timeout=0, elapsed=344), Traces: [0ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=null) with no deferred attached!
06:34:28.333 [DEBUG - kudu-nio-0] (KuduRpc.java:260) Handling a callback on RPC KuduRpc(method=ConnectToMaster, tablet=null, attempt=0, TimeoutTracker(timeout=0, elapsed=64), Traces: [0ms] received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=null) with no deferred attached!
06:34:28.333 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.333 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.348 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.349 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.349 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.349 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.349 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.349 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.349 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.350 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.369 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.369 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.369 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.369 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.369 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.369 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.369 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.371 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.388 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.389 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.389 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.389 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.389 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.389 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.389 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.390 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.409 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.409 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.409 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.409 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.409 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.409 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.409 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.410 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.428 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.429 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.429 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.429 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.429 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.429 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.429 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.430 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.448 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.449 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.449 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.449 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.449 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.449 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.449 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.449 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.450 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.488 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.489 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.489 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.489 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.489 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.489 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.489 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.490 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.509 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.509 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.509 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.509 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.509 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.510 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.510 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.511 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.511 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1437959684(state=PENDING, result=null, callback=(continuation of Deferred@703479603 after retry RPC after error@1066295810) -> (continuation of Deferred@341985152 after retry RPC after error@1839101614) -> (continuation of Deferred@902945673 after retry RPC after error@2038636867) -> (continuation of Deferred@2136684468 after retry RPC after error@1634600439) -> (continuation of Deferred@1515751360 after retry RPC after error@984689905) -> (continuation of Deferred@477186692 after retry RPC after error@228697408) -> (continuation of Deferred@333006259 after retry RPC after error@540816797), errback=(continuation of Deferred@703479603 after retry RPC after error@1066295810) -> (continuation of Deferred@341985152 after retry RPC after error@1839101614) -> (continuation of Deferred@902945673 after retry RPC after error@2038636867) -> (continuation of Deferred@2136684468 after retry RPC after error@1634600439) -> (continuation of Deferred@1515751360 after retry RPC after error@984689905) -> (continuation of Deferred@477186692 after retry RPC after error@228697408) -> (continuation of Deferred@333006259 after retry RPC after error@540816797)))
06:34:28.512 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=8, TimeoutTracker(timeout=250, elapsed=222), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [45ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [83ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [83ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [100ms] refreshing cache from master, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [100ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [101ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [102ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [102ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [120ms] refreshing cache from master, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] refreshing cache from master, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [223ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1437959684(state=PENDING, result=null, callback=(continuation of Deferred@703479603 after retry RPC after error@1066295810) -> (continuation of Deferred@341985152 after retry RPC after error@1839101614) -> (continuation of Deferred@902945673 after retry RPC after error@2038636867) -> (continuation of Deferred@2136684468 after retry RPC after error@1634600439) -> (continuation of Deferred@1515751360 after retry RPC after error@984689905) -> (continuation of Deferred@477186692 after retry RPC after error@228697408) -> (continuation of Deferred@333006259 after retry RPC after error@540816797), errback=(continuation of Deferred@703479603 after retry RPC after error@1066295810) -> (continuation of Deferred@341985152 after retry RPC after error@1839101614) -> (continuation of Deferred@902945673 after retry RPC after error@2038636867) -> (continuation of Deferred@2136684468 after retry RPC after error@1634600439) -> (continuation of Deferred@1515751360 after retry RPC after error@984689905) -> (continuation of Deferred@477186692 after retry RPC after error@228697408) -> (continuation of Deferred@333006259 after retry RPC after error@540816797)))
06:34:28.528 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.529 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.529 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.529 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.529 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.529 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.529 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.529 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.530 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.548 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.548 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.548 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.549 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.549 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.549 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.549 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.550 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.568 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.569 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.569 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.569 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.569 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.569 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.569 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.569 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.570 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.588 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.589 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.589 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.589 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.589 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.589 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.589 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.590 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.608 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.609 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.609 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.609 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.609 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.609 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.609 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.609 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.610 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.648 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.649 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.649 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.649 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.649 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.649 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.649 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.650 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.688 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.689 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.689 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.689 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.689 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.689 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.689 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.690 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.690 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1302160147(state=PENDING, result=null, callback=(continuation of Deferred@1457438121 after retry RPC after error@1972961651) -> (continuation of Deferred@339949634 after retry RPC after error@853558306) -> (continuation of Deferred@300975138 after retry RPC after error@752848374) -> (continuation of Deferred@559100167 after retry RPC after error@295874651) -> (continuation of Deferred@82241497 after retry RPC after error@1409706638) -> (continuation of Deferred@1826992640 after retry RPC after error@1749153711), errback=(continuation of Deferred@1457438121 after retry RPC after error@1972961651) -> (continuation of Deferred@339949634 after retry RPC after error@853558306) -> (continuation of Deferred@300975138 after retry RPC after error@752848374) -> (continuation of Deferred@559100167 after retry RPC after error@295874651) -> (continuation of Deferred@82241497 after retry RPC after error@1409706638) -> (continuation of Deferred@1826992640 after retry RPC after error@1749153711)))
06:34:28.691 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=161), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [2ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [20ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [43ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [160ms] refreshing cache from master, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [160ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [161ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [162ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [162ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, deferred=Deferred@1302160147(state=PENDING, result=null, callback=(continuation of Deferred@1457438121 after retry RPC after error@1972961651) -> (continuation of Deferred@339949634 after retry RPC after error@853558306) -> (continuation of Deferred@300975138 after retry RPC after error@752848374) -> (continuation of Deferred@559100167 after retry RPC after error@295874651) -> (continuation of Deferred@82241497 after retry RPC after error@1409706638) -> (continuation of Deferred@1826992640 after retry RPC after error@1749153711), errback=(continuation of Deferred@1457438121 after retry RPC after error@1972961651) -> (continuation of Deferred@339949634 after retry RPC after error@853558306) -> (continuation of Deferred@300975138 after retry RPC after error@752848374) -> (continuation of Deferred@559100167 after retry RPC after error@295874651) -> (continuation of Deferred@82241497 after retry RPC after error@1409706638) -> (continuation of Deferred@1826992640 after retry RPC after error@1749153711)))
06:34:28.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.709 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.709 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.709 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.709 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.710 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.710 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.710 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.711 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.728 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.729 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.729 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.729 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.729 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.729 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.729 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.730 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.748 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.748 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.748 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.749 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.749 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.749 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.749 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.750 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.768 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.769 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.769 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.769 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.769 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.769 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.769 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.769 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.771 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.788 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.789 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.789 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.789 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.789 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.789 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.789 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.790 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.828 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.829 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.829 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.829 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.829 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.829 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.829 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.830 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.848 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.849 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.849 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.849 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.849 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.849 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.849 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.849 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.850 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.908 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.909 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.909 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.909 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.909 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.909 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.909 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.910 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.928 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.928 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.929 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.929 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.929 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.929 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.929 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.930 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.930 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1235543349(state=PENDING, result=null, callback=(continuation of Deferred@1100193622 after retry RPC after error@1853109647) -> (continuation of Deferred@1111474475 after retry RPC after error@752617748) -> (continuation of Deferred@1911048950 after retry RPC after error@1056952585) -> (continuation of Deferred@41249124 after retry RPC after error@1311593623) -> (continuation of Deferred@2107705438 after retry RPC after error@1033304442) -> (continuation of Deferred@335649253 after retry RPC after error@808255006) -> (continuation of Deferred@1161491389 after retry RPC after error@1332580169) -> (continuation of Deferred@1471937008 after retry RPC after error@11542807), errback=(continuation of Deferred@1100193622 after retry RPC after error@1853109647) -> (continuation of Deferred@1111474475 after retry RPC after error@752617748) -> (continuation of Deferred@1911048950 after retry RPC after error@1056952585) -> (continuation of Deferred@41249124 after retry RPC after error@1311593623) -> (continuation of Deferred@2107705438 after retry RPC after error@1033304442) -> (continuation of Deferred@335649253 after retry RPC after error@808255006) -> (continuation of Deferred@1161491389 after retry RPC after error@1332580169) -> (continuation of Deferred@1471937008 after retry RPC after error@11542807)))
06:34:28.931 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=9, TimeoutTracker(timeout=250, elapsed=221), Traces: [0ms] refreshing cache from master, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [21ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [42ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [42ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [61ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [63ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [122ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [140ms] refreshing cache from master, [140ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [141ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [141ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [142ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [142ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [220ms] refreshing cache from master, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [220ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [221ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [222ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [222ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1235543349(state=PENDING, result=null, callback=(continuation of Deferred@1100193622 after retry RPC after error@1853109647) -> (continuation of Deferred@1111474475 after retry RPC after error@752617748) -> (continuation of Deferred@1911048950 after retry RPC after error@1056952585) -> (continuation of Deferred@41249124 after retry RPC after error@1311593623) -> (continuation of Deferred@2107705438 after retry RPC after error@1033304442) -> (continuation of Deferred@335649253 after retry RPC after error@808255006) -> (continuation of Deferred@1161491389 after retry RPC after error@1332580169) -> (continuation of Deferred@1471937008 after retry RPC after error@11542807), errback=(continuation of Deferred@1100193622 after retry RPC after error@1853109647) -> (continuation of Deferred@1111474475 after retry RPC after error@752617748) -> (continuation of Deferred@1911048950 after retry RPC after error@1056952585) -> (continuation of Deferred@41249124 after retry RPC after error@1311593623) -> (continuation of Deferred@2107705438 after retry RPC after error@1033304442) -> (continuation of Deferred@335649253 after retry RPC after error@808255006) -> (continuation of Deferred@1161491389 after retry RPC after error@1332580169) -> (continuation of Deferred@1471937008 after retry RPC after error@11542807)))
06:34:28.948 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.949 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.949 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.949 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.949 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.949 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.949 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.949 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.951 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.968 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.969 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.969 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.969 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.969 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.969 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.969 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.970 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:28.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:28.988 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:28.988 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.988 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.989 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.989 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.989 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:28.989 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:28.989 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.008 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.009 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.009 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.009 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.009 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.009 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.009 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.010 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.028 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.028 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.028 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.028 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.028 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.029 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.029 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.030 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.068 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.069 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.069 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.069 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.069 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.069 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.069 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.069 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.070 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.148 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.149 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.149 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.149 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.149 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.149 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.149 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.149 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.150 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.151 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1189418565(state=PENDING, result=null, callback=(continuation of Deferred@70005551 after retry RPC after error@1121726355) -> (continuation of Deferred@1266310739 after retry RPC after error@625163847) -> (continuation of Deferred@1075037323 after retry RPC after error@1670433709) -> (continuation of Deferred@2055313738 after retry RPC after error@1175200091) -> (continuation of Deferred@409305741 after retry RPC after error@2110648207) -> (continuation of Deferred@200303285 after retry RPC after error@751934569), errback=(continuation of Deferred@70005551 after retry RPC after error@1121726355) -> (continuation of Deferred@1266310739 after retry RPC after error@625163847) -> (continuation of Deferred@1075037323 after retry RPC after error@1670433709) -> (continuation of Deferred@2055313738 after retry RPC after error@1175200091) -> (continuation of Deferred@409305741 after retry RPC after error@2110648207) -> (continuation of Deferred@200303285 after retry RPC after error@751934569)))
06:34:29.151 [DEBUG - kudu-nio-0] (KuduTransaction.java:796) sending keepalive message after prior one timed out (txn ID 0): org.apache.kudu.client.NonRecoverableException: cannot complete before timeout: KuduRpc(method=KeepTransactionAlive, tablet=null, attempt=7, TimeoutTracker(timeout=250, elapsed=202), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [20ms] refreshing cache from master, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [20ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [21ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [22ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [22ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [40ms] refreshing cache from master, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [40ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [41ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [41ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [60ms] refreshing cache from master, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [60ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [61ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [62ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [62ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [80ms] refreshing cache from master, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [80ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [81ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [82ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [82ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [120ms] refreshing cache from master, [120ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [121ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [121ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [122ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [123ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [200ms] refreshing cache from master, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [200ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [201ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [202ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [202ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, deferred=Deferred@1189418565(state=PENDING, result=null, callback=(continuation of Deferred@70005551 after retry RPC after error@1121726355) -> (continuation of Deferred@1266310739 after retry RPC after error@625163847) -> (continuation of Deferred@1075037323 after retry RPC after error@1670433709) -> (continuation of Deferred@2055313738 after retry RPC after error@1175200091) -> (continuation of Deferred@409305741 after retry RPC after error@2110648207) -> (continuation of Deferred@200303285 after retry RPC after error@751934569), errback=(continuation of Deferred@70005551 after retry RPC after error@1121726355) -> (continuation of Deferred@1266310739 after retry RPC after error@625163847) -> (continuation of Deferred@1075037323 after retry RPC after error@1670433709) -> (continuation of Deferred@2055313738 after retry RPC after error@1175200091) -> (continuation of Deferred@409305741 after retry RPC after error@2110648207) -> (continuation of Deferred@200303285 after retry RPC after error@751934569)))
06:34:29.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.168 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.168 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.168 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.168 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.168 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.168 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.169 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.170 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.188 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.189 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.189 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.189 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.189 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.189 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.189 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.189 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.190 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.208 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.209 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.209 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.209 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.209 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.209 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.209 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.210 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.229 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.229 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.229 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.229 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.230 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.230 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.230 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.231 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.252:42373(127.23.161.252:42373)] connecting to peer
06:34:29.248 [DEBUG - pool-49-thread-1] (Connection.java:748) [peer master-127.23.161.254:38015(127.23.161.254:38015)] connecting to peer
06:34:29.249 [INFO - kudu-nio-1] (Connection.java:429) Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.249 [DEBUG - kudu-nio-1] (Connection.java:709) [peer master-127.23.161.254:38015(127.23.161.254:38015)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.249 [INFO - kudu-nio-1] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.254:38015: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015
06:34:29.249 [INFO - kudu-nio-0] (Connection.java:429) Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.250 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.252:42373(127.23.161.252:42373)] cleaning up while in state CONNECTING due to: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.250 [INFO - kudu-nio-0] (ConnectToCluster.java:385) Unable to connect to master 127.23.161.252:42373: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373
06:34:29.251 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.252 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:1887) Cannot continue with RPC because of: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=13, TimeoutTracker(timeout=50000, elapsed=49946), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [6ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [25ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [25ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [26ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [27ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [45ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [46ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [47ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] Sub RPC ConnectToMaster: refreshing cache from master, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [67ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [67ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: refreshing cache from master, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [86ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [104ms] Sub RPC ConnectToMaster: refreshing cache from master, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [106ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: refreshing cache from master, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [126ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [126ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: refreshing cache from master, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [166ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [166ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: refreshing cache from master, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [188ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [188ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [264ms] Sub RPC ConnectToMaster: refreshing cache from master, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [265ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [266ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [267ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [267ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [344ms] Sub RPC ConnectToMaster: refreshing cache from master, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [348ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [348ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [544ms] Sub RPC ConnectToMaster: refreshing cache from master, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [546ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [546ms] too many traces: truncated at 100 traces, deferred=Deferred@2036130395(state=PENDING, result=null, callback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248), errback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248)))
06:34:29.253 [WARN - kudu-nio-0] (ConnectToCluster.java:305) Unable to find the leader master 127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177; will retry
06:34:29.254 [DEBUG - main] (Connection.java:575) [peer 93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)] disconnecting while in state READY
06:34:29.254 [DEBUG - main] (Connection.java:575) [peer master-127.23.161.254:38015(127.23.161.254:38015)] disconnecting while in state TERMINATED
06:34:29.254 [DEBUG - main] (Connection.java:575) [peer master-127.23.161.252:42373(127.23.161.252:42373)] disconnecting while in state TERMINATED
06:34:29.254 [DEBUG - main] (Connection.java:575) [peer master-127.23.161.253:45177(127.23.161.253:45177)] disconnecting while in state READY
06:34:29.256 [DEBUG - kudu-nio-1] (Connection.java:239) [peer 93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)] handling channelInactive
06:34:29.256 [DEBUG - kudu-nio-0] (AsyncKuduClient.java:2704) Releasing all remaining resources
06:34:29.256 [DEBUG - kudu-nio-1] (Connection.java:709) [peer 93e8c8ec161e4104b5dc58fa5a609f99(127.23.161.194:45449)] cleaning up while in state READY due to: connection closed
06:34:29.257 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.257072 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24525
06:34:29.260 [DEBUG - kudu-nio-0] (Connection.java:239) [peer master-127.23.161.253:45177(127.23.161.253:45177)] handling channelInactive
06:34:29.260 [DEBUG - kudu-nio-0] (Connection.java:709) [peer master-127.23.161.253:45177(127.23.161.253:45177)] cleaning up while in state READY due to: connection closed
06:34:29.267 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.267565 24697 generic_service.cc:196] Checking for leaks (request via RPC)
06:34:29.379 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.379104 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24525
06:34:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.409490 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24708
06:34:29.409 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:29.409541 24725 connection.cc:535] client connection to 127.23.161.193:37599 recv error: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:34:29.410 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:29.409802 24725 proxy.cc:239] Call had error, refreshing address and retrying: Network error: recv error from unknown peer: Transport endpoint is not connected (error 107)
06:34:29.414 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:29.413884 24725 consensus_peers.cc:487] T 22852a0334cf48fbb5c4242fe8272a67 P 93e8c8ec161e4104b5dc58fa5a609f99 -> Peer 56f0ce12c25a4723a04412921d35ce43 (127.23.161.193:37599): Couldn't send request to peer 56f0ce12c25a4723a04412921d35ce43. Status: Network error: Client connection negotiation failed: client connection to 127.23.161.193:37599: connect: Connection refused (error 111). This is attempt 1: this message will repeat every 5th retry.
06:34:29.419 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.419106 24852 generic_service.cc:196] Checking for leaks (request via RPC)
06:34:29.533 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.533758 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24708
06:34:29.559 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.559162 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 24862
06:34:29.567 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.566903 25006 generic_service.cc:196] Checking for leaks (request via RPC)
06:34:29.726 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) W20260809 06:34:29.726629 24878 proxy.cc:239] Call had error, refreshing address and retrying: Network error: Client connection negotiation failed: client connection to 127.23.161.193:37599: connect: Connection refused (error 111)
06:34:29.727 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.727449 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 24862
06:34:29.762 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.762236 24199 external_mini_cluster.cc:1636] Attempting to check leaks for /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu pid 25314
06:34:29.768 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.768707 25397 generic_service.cc:196] Checking for leaks (request via RPC)
06:34:29.928 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) I20260809 06:34:29.926802 24199 external_mini_cluster.cc:1595] Killing /tmp/dist-test-taskbZXLUY/build/asan/bin/kudu with pid 25314
06:34:29.991 [INFO - cluster stderr printer] (MiniKuduCluster.java:751) 2026-08-09T06:34:29Z chronyd exiting
06:34:30.101 [ERROR - main] (RetryRule.java:206) org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager: failed attempt 1
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=13, TimeoutTracker(timeout=50000, elapsed=49946), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [6ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [25ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [25ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [26ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [27ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [45ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [46ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [47ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] Sub RPC ConnectToMaster: refreshing cache from master, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [67ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [67ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: refreshing cache from master, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [86ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [104ms] Sub RPC ConnectToMaster: refreshing cache from master, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [106ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: refreshing cache from master, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [126ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [126ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: refreshing cache from master, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [166ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [166ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: refreshing cache from master, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [188ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [188ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [264ms] Sub RPC ConnectToMaster: refreshing cache from master, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [265ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [266ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [267ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [267ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [344ms] Sub RPC ConnectToMaster: refreshing cache from master, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [348ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [348ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [544ms] Sub RPC ConnectToMaster: refreshing cache from master, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [546ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [546ms] too many traces: truncated at 100 traces, deferred=Deferred@2036130395(state=PENDING, result=null, callback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248), errback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248)))
at org.junit.Assert.fail(Assert.java:89) ~[junit-4.13.2.jar:4.13.2]
at org.junit.Assert.assertTrue(Assert.java:42) ~[junit-4.13.2.jar:4.13.2]
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214) ~[test/:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_362]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_362]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_362]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_362]
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2]
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2]
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) ~[?:1.8.0_362]
at java.lang.Thread.run(Thread.java:750) ~[?:1.8.0_362]
java.lang.AssertionError: Timed out: cannot complete before timeout: KuduRpc(method=GetTransactionState, tablet=null, attempt=13, TimeoutTracker(timeout=50000, elapsed=49946), Traces: [0ms] refreshing cache from master, [0ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [1ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [1ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [2ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [3ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [3ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [4ms] refreshing cache from master, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [4ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [5ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [6ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [6ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [25ms] refreshing cache from master, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [25ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [25ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [26ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [27ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [27ms] delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [45ms] refreshing cache from master, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [45ms] Sub RPC ConnectToMaster: delaying RPC due to: Illegal state: connection is terminated, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [45ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [46ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [47ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [64ms] Sub RPC ConnectToMaster: refreshing cache from master, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [65ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [66ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [66ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [67ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [67ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [84ms] Sub RPC ConnectToMaster: refreshing cache from master, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [84ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [85ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [85ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [86ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [86ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [104ms] Sub RPC ConnectToMaster: refreshing cache from master, [104ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [105ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [105ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [106ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [106ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [124ms] Sub RPC ConnectToMaster: refreshing cache from master, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [124ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [125ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [125ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [126ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [126ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: refreshing cache from master, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [164ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [165ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [166ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [166ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: refreshing cache from master, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [186ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [186ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [188ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [188ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [264ms] Sub RPC ConnectToMaster: refreshing cache from master, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [264ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [265ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [265ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [266ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [267ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [267ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [344ms] Sub RPC ConnectToMaster: refreshing cache from master, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [345ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [346ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.254:38015: Network error: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [348ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.253:45177: OK, [348ms] Sub RPC ConnectToMaster: delaying RPC due to: Service unavailable: Master config (127.23.161.252:42373,127.23.161.254:38015,127.23.161.253:45177) has no leader. Exceptions received: org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373,org.apache.kudu.client.RecoverableException: Failed to connect to peer master-127.23.161.254:38015(127.23.161.254:38015): Connection refused: /127.23.161.254:38015, [544ms] Sub RPC ConnectToMaster: refreshing cache from master, [544ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.252:42373, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.254:38015, [545ms] Sub RPC ConnectToMaster: sending RPC to server master-127.23.161.253:45177, [546ms] Sub RPC ConnectToMaster: received response from server master-127.23.161.252:42373: Network error: Failed to connect to peer master-127.23.161.252:42373(127.23.161.252:42373): Connection refused: /127.23.161.252:42373, [546ms] too many traces: truncated at 100 traces, deferred=Deferred@2036130395(state=PENDING, result=null, callback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248), errback=(continuation of Deferred@875937366 after retry RPC after error@1305719769) -> (continuation of Deferred@1483763509 after retry RPC after error@1184895066) -> (continuation of Deferred@1602142678 after retry RPC after error@266595483) -> (continuation of Deferred@1412506517 after retry RPC after error@316005257) -> (continuation of Deferred@999086196 after retry RPC after error@723217578) -> (continuation of Deferred@1986777047 after retry RPC after error@1256086272) -> (continuation of Deferred@399275351 after retry RPC after error@1732388152) -> (continuation of Deferred@1333742448 after retry RPC after error@2000756676) -> (continuation of Deferred@64207542 after retry RPC after error@1976443921) -> (continuation of Deferred@1080022291 after retry RPC after error@1067408867) -> (continuation of Deferred@562517676 after retry RPC after error@636293153) -> (continuation of Deferred@463781570 after retry RPC after error@1578207248)))
at org.junit.Assert.fail(Assert.java:89)
at org.junit.Assert.assertTrue(Assert.java:42)
at org.apache.kudu.client.TestKuduTransaction.testTxnKeepaliveRollingSwitchToOtherTxnManager(TestKuduTransaction.java:1214)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299)
at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.lang.Thread.run(Thread.java:750)